http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java new file mode 100644 index 0000000..0344305 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/DontSplitArguments.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.converters.IParameterSplitter; + +import java.util.ArrayList; +import java.util.List; + +public class DontSplitArguments implements IParameterSplitter { + + @Override + public List<String> split(String value) { + List<String> list = new ArrayList<>(1); + list.add(value); + return list; + } +}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java new file mode 100644 index 0000000..7524053 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsAccessor.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import java.io.File; + +/** + * Launch args for create and start and anything else that can start something + */ +public interface LaunchArgsAccessor extends WaitTimeAccessor { + String getRmAddress(); + + File getOutputFile(); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java new file mode 100644 index 0000000..bc7e94c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/LaunchArgsDelegate.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.Parameter; + +import java.io.File; + +/** + * Any launch-time args + */ +public class LaunchArgsDelegate extends WaitArgsDelegate implements + LaunchArgsAccessor { + + + //TODO: do we need this? + @Parameter(names = ARG_RESOURCE_MANAGER, + description = "Resource manager hostname:port ", + required = false) + private String rmAddress; + + @Override + public String getRmAddress() { + return rmAddress; + } + + @Parameter(names = {ARG_OUTPUT, ARG_OUTPUT_SHORT}, + description = "output file for any application report") + public File outputFile; + + @Override + public File getOutputFile() { + return outputFile; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java new file mode 100644 index 0000000..ccb526c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/PathArgumentConverter.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.converters.BaseConverter; +import org.apache.hadoop.fs.Path; + +public class PathArgumentConverter extends BaseConverter<Path> { + + public PathArgumentConverter(String optionName) { + super(optionName); + } + + @Override + public Path convert(String value) { + return new Path(value); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java new file mode 100644 index 0000000..f9516d1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMArgs.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.slider.common.params; + +/** + * Parameters sent by the Client to the AM + */ +public class SliderAMArgs extends CommonArgs { + + SliderAMCreateAction createAction = new SliderAMCreateAction(); + + public SliderAMArgs(String[] args) { + super(args); + } + + @Override + protected void addActionArguments() { + addActions(createAction); + } + + public String getImage() { + return createAction.image; + } + + /** + * This is the URI in the FS to the Slider cluster; the conf file (and any + * other cluster-specifics) can be picked up here + */ + public String getSliderClusterURI() { + return createAction.sliderClusterURI; + } + + /** + * Am binding is simple: there is only one action + */ + @Override + public void applyAction() { + bindCoreAction(createAction); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java new file mode 100644 index 0000000..197c22b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderAMCreateAction.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import com.beust.jcommander.ParametersDelegate; + +import java.io.File; + + +@Parameters(commandNames = {SliderActions.ACTION_CREATE}, + commandDescription = SliderActions.DESCRIBE_ACTION_CREATE) + +public class SliderAMCreateAction extends AbstractActionArgs implements + LaunchArgsAccessor { + + + @Override + public String getActionName() { + return SliderActions.ACTION_CREATE; + } + + @Parameter(names = ARG_IMAGE, description = "image", required = false) + public String image; + + /** + * This is the URI in the FS to the Slider cluster; the conf file (and any + * other cluster-specifics) can be picked up here + */ + @Parameter(names = ARG_CLUSTER_URI, + description = "URI to the Slider cluster", required = true) + public String sliderClusterURI; + + @ParametersDelegate + LaunchArgsDelegate launchArgs = new LaunchArgsDelegate(); + + @Override + public String getRmAddress() { + return launchArgs.getRmAddress(); + } + + @Override + public int getWaittime() { + return launchArgs.getWaittime(); + } + + @Override + public void setWaittime(int waittime) { + launchArgs.setWaittime(waittime); + } + + @Override + public File getOutputFile() { + return launchArgs.getOutputFile(); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java new file mode 100644 index 0000000..aab7c98 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +/** + * Actions. + * Only some of these are supported by specific Slider Services; they + * are listed here to ensure the names are consistent + */ +public interface SliderActions { + String ACTION_AM_SUICIDE = "am-suicide"; + String ACTION_BUILD = "build"; + String ACTION_CLIENT = "client"; + String ACTION_CREATE = "create"; + String ACTION_DIAGNOSTICS = "diagnostics"; + String ACTION_DEPENDENCY = "dependency"; + String ACTION_UPDATE = "update"; + String ACTION_UPGRADE = "upgrade"; + String ACTION_DESTROY = "destroy"; + String ACTION_ECHO = "echo"; + String ACTION_EXISTS = "exists"; + String ACTION_FLEX = "flex"; + String ACTION_FREEZE = "stop"; + String ACTION_HELP = "help"; + String ACTION_INSTALL_KEYTAB = "install-keytab"; + String ACTION_INSTALL_PACKAGE = "install-package"; + String ACTION_KDIAG = "kdiag"; + String ACTION_KEYTAB = "keytab"; + String ACTION_KILL_CONTAINER = "kill-container"; + String ACTION_LIST = "list"; + String ACTION_LOOKUP = "lookup"; + String ACTION_NODES = "nodes"; + String ACTION_PACKAGE = "package"; + String ACTION_PREFLIGHT = "preflight"; + String ACTION_RECONFIGURE = "reconfigure"; + String ACTION_REGISTRY = "registry"; + String ACTION_RESOLVE = "resolve"; + String ACTION_STATUS = "status"; + String ACTION_THAW = "start"; + String ACTION_TOKENS = "tokens"; + + String ACTION_VERSION = "version"; + String DESCRIBE_ACTION_AM_SUICIDE = + "Tell the Slider Application Master to simulate a process failure by terminating itself"; + String DESCRIBE_ACTION_BUILD = + "Build a Slider cluster specification, but do not start it"; + String DESCRIBE_ACTION_CREATE = + "Create a live Slider application"; + String DESCRIBE_ACTION_DEPENDENCY = + "Slider AM and agent dependency (libraries) management"; + String DESCRIBE_ACTION_UPDATE = + "Update template for a Slider application"; + String DESCRIBE_ACTION_UPGRADE = + "Rolling upgrade/downgrade the application to a newer/previous version"; + String DESCRIBE_ACTION_DESTROY = + "Destroy a stopped Slider application"; + String DESCRIBE_ACTION_EXISTS = + "Probe for an application running"; + String DESCRIBE_ACTION_FLEX = "Flex a Slider application"; + String DESCRIBE_ACTION_FREEZE = + "Stop a running application"; + String DESCRIBE_ACTION_GETCONF = + "Get the configuration of an application"; + String DESCRIBE_ACTION_KDIAG = "Diagnose Kerberos problems"; + String DESCRIBE_ACTION_KILL_CONTAINER = + "Kill a container in the application"; + String DESCRIBE_ACTION_HELP = "Print help information"; + String DESCRIBE_ACTION_LIST = + "List running Slider applications"; + String DESCRIBE_ACTION_LOOKUP = + "look up a YARN application"; + String DESCRIBE_ACTION_NODES = "List the node information for the YARN cluster or a running application"; + String DESCRIBE_ACTION_MONITOR = + "Monitor a running application"; + String DESCRIBE_ACTION_REGISTRY = + "Query the registry of a YARN application"; + String DESCRIBE_ACTION_RESOLVE = + "Resolve or list records in the YARN registry"; + String DESCRIBE_ACTION_STATUS = + "Get the status of an application"; + String DESCRIBE_ACTION_THAW = + "Start a stopped application"; + String DESCRIBE_ACTION_VERSION = + "Print the Slider version information"; + String DESCRIBE_ACTION_INSTALL_PACKAGE = "Install application package." + + " Deprecated, use '" + ACTION_PACKAGE + " " + ClientArgs.ARG_INSTALL + "'."; + String DESCRIBE_ACTION_PACKAGE = "Install/list/delete application packages and list app instances that use the packages"; + String DESCRIBE_ACTION_CLIENT = "Install the application client in the specified directory or obtain a client keystore or truststore"; + String DESCRIBE_ACTION_INSTALL_KEYTAB = "Install the Kerberos keytab." + + " Deprecated, use '" + ACTION_KEYTAB + " " + ClientArgs.ARG_INSTALL + "'."; + String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory"; + String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client"; + +} + http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java new file mode 100644 index 0000000..b0d1ebf --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URIArgumentConverter.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.converters.BaseConverter; + +import java.net.URI; +import java.net.URISyntaxException; + +public class URIArgumentConverter extends BaseConverter<URI> { + + public URIArgumentConverter(String optionName) { + super(optionName); + } + + @Override + public URI convert(String value) { + try { + return new URI(value); + } catch (URISyntaxException e) { + throw new RuntimeException("Cannot make a URI from " + value); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java new file mode 100644 index 0000000..8894309 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/URLArgumentConverter.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.slider.common.params; + +import com.beust.jcommander.converters.BaseConverter; + +import java.net.MalformedURLException; +import java.net.URL; + +public class URLArgumentConverter extends BaseConverter<URL> { + public URLArgumentConverter(String optionName) { + super(optionName); + } + + @Override + public URL convert(String value) { + try { + return new URL(value); + } catch (MalformedURLException e) { + throw new RuntimeException("Cannot make a URL from " + value); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java new file mode 100644 index 0000000..1c27c01 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitArgsDelegate.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +import com.beust.jcommander.Parameter; + +public class WaitArgsDelegate extends AbstractArgsDelegate implements + WaitTimeAccessor { + + + //--wait [timeout] + @Parameter(names = {ARG_WAIT}, + description = "time to wait for an action to complete") + public int waittime = 0; + + + @Override + public int getWaittime() { + return waittime; + } + + @Override + public void setWaittime(int waittime) { + this.waittime = waittime; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java new file mode 100644 index 0000000..13d4d5a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/params/WaitTimeAccessor.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.params; + +public interface WaitTimeAccessor { + int getWaittime(); + void setWaittime(int waittime); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java new file mode 100644 index 0000000..6380d0c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/Comparators.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.tools; + +import java.io.Serializable; +import java.util.Comparator; + +/** + * Some general comparators + */ +public class Comparators { + + public static class LongComparator implements Comparator<Long>, Serializable { + @Override + public int compare(Long o1, Long o2) { + long result = o1 - o2; + // need to comparisons with a diff greater than integer size + if (result < 0 ) { + return -1; + } else if (result > 0) { + return 1; + } + return 0; + } + } +public static class InvertedLongComparator implements Comparator<Long>, Serializable { + private static final LongComparator inner = new LongComparator(); + @Override + public int compare(Long o1, Long o2) { + return -inner.compare(o1, o2); + } + } + + + /** + * Little template class to reverse any comparitor + * @param <CompareType> the type that is being compared + */ + public static class ComparatorReverser<CompareType> implements Comparator<CompareType>, + Serializable { + + final Comparator<CompareType> instance; + + public ComparatorReverser(Comparator<CompareType> instance) { + this.instance = instance; + } + + @Override + public int compare(CompareType first, CompareType second) { + return instance.compare(second, first); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java new file mode 100644 index 0000000..9db241d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java @@ -0,0 +1,658 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.tools; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.registry.client.api.RegistryConstants; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.SliderXmlConfKeys; +import org.apache.slider.core.exceptions.BadConfigException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.w3c.dom.Document; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.StringWriter; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Methods to aid in config, both in the Configuration class and + * with other parts of setting up Slider-initated processes. + * + * Some of the methods take an argument of a map iterable for their sources; this allows + * the same method + */ +public class ConfigHelper { + private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class); + + private static AtomicBoolean sliderResourceInjected = + new AtomicBoolean(false); + private static AtomicBoolean sliderResourceInjectionAttempted = + new AtomicBoolean(false); + + /** + * Dump the (sorted) configuration + * @param conf config + * @return the sorted keyset + */ + public static Set<String> dumpConf(Configuration conf) { + Set<String> keys = sortedConfigKeys(conf); + for (String key : keys) { + log.info("{}={}", key, conf.get(key)); + } + return keys; + } + + /** + * Take a configuration and return a sorted set + * @param conf config + * @return the sorted keyset + + */ + public static Set<String> sortedConfigKeys(Iterable<Map.Entry<String, String>> conf) { + TreeSet<String> sorted = new TreeSet<String>(); + for (Map.Entry<String, String> entry : conf) { + sorted.add(entry.getKey()); + } + return sorted; + } + + /** + * Set an entire map full of values + * + * @param config config to patch + * @param map map of data + * @param origin origin data + */ + public static void addConfigMap(Configuration config, + Map<String, String> map, + String origin) throws BadConfigException { + addConfigMap(config, map.entrySet(), origin); + } + + /** + * Set an entire map full of values + * + * @param config config to patch + * @param map map of data + * @param origin origin data + */ + public static void addConfigMap(Configuration config, + Iterable<Map.Entry<String, String>> map, + String origin) throws BadConfigException { + for (Map.Entry<String, String> mapEntry : map) { + String key = mapEntry.getKey(); + String value = mapEntry.getValue(); + if (value == null) { + throw new BadConfigException("Null value for property " + key); + } + config.set(key, value, origin); + } + } + + + /** + * Save a config file in a destination directory on a given filesystem + * @param systemConf system conf used for creating filesystems + * @param confToSave config to save + * @param confdir the directory path where the file is to go + * @param filename the filename + * @return the destination path where the file was saved + * @throws IOException IO problems + */ + public static Path saveConfig(Configuration systemConf, + Configuration confToSave, + Path confdir, + String filename) throws IOException { + FileSystem fs = FileSystem.get(confdir.toUri(), systemConf); + Path destPath = new Path(confdir, filename); + saveConfig(fs, destPath, confToSave); + return destPath; + } + + /** + * Save a config + * @param fs filesystem + * @param destPath dest to save + * @param confToSave config to save + * @throws IOException IO problems + */ + public static void saveConfig(FileSystem fs, + Path destPath, + Configuration confToSave) throws + IOException { + FSDataOutputStream fos = fs.create(destPath); + try { + confToSave.writeXml(fos); + } finally { + IOUtils.closeStream(fos); + } + } + + /** + * Convert to an XML string + * @param conf configuration + * @return conf + * @throws IOException + */ + public static String toXml(Configuration conf) throws IOException { + StringWriter writer = new StringWriter(); + conf.writeXml(writer); + return writer.toString(); + } + + /** + * This will load and parse a configuration to an XML document + * @param fs filesystem + * @param path path + * @return an XML document + * @throws IOException IO failure + */ + public Document parseConfiguration(FileSystem fs, + Path path) throws + IOException { + + + byte[] data = loadBytes(fs, path); + //this is here to track down a parse issue + //related to configurations + String s = new String(data, 0, data.length); + log.debug("XML resource {} is \"{}\"", path, s); +/* JDK7 + try (ByteArrayInputStream in = new ByteArrayInputStream(data)) { + Document document = parseConfigXML(in); + return document; + } catch (ParserConfigurationException | SAXException e) { + throw new IOException(e); + } +*/ + ByteArrayInputStream in= null; + try { + in = new ByteArrayInputStream(data); + Document document = parseConfigXML(in); + return document; + } catch (ParserConfigurationException e) { + throw new IOException(e); + } catch (SAXException e) { + throw new IOException(e); + } finally { + IOUtils.closeStream(in); + } + } + + public static byte[] loadBytes(FileSystem fs, Path path) throws IOException { + int len = (int) fs.getLength(path); + byte[] data = new byte[len]; + /* JDK7 + try(FSDataInputStream in = fs.open(path)) { + in.readFully(0, data); + } +*/ + FSDataInputStream in = null; + in = fs.open(path); + try { + in.readFully(0, data); + } finally { + IOUtils.closeStream(in); + } + return data; + } + + /** + * Load a configuration from ANY FS path. The normal Configuration + * loader only works with file:// URIs + * @param fs filesystem + * @param path path + * @return a loaded resource + * @throws IOException + */ + public static Configuration loadConfiguration(FileSystem fs, + Path path) throws IOException { + byte[] data = loadBytes(fs, path); + + ByteArrayInputStream in2; + + in2 = new ByteArrayInputStream(data); + Configuration conf1 = new Configuration(false); + conf1.addResource(in2); + //now clone it while dropping all its sources + Configuration conf2 = new Configuration(false); + String src = path.toString(); + for (Map.Entry<String, String> entry : conf1) { + String key = entry.getKey(); + String value = entry.getValue(); + conf2.set(key, value, src); + } + return conf2; + } + + + /** + * Generate a config file in a destination directory on the local filesystem + * @param confdir the directory path where the file is to go + * @param filename the filename + * @return the destination path + */ + public static File saveConfig(Configuration generatingConf, + File confdir, + String filename) throws IOException { + + + File destPath = new File(confdir, filename); + OutputStream fos = new FileOutputStream(destPath); + try { + generatingConf.writeXml(fos); + } finally { + IOUtils.closeStream(fos); + } + return destPath; + } + + /** + * Parse an XML Hadoop configuration into an XML document. x-include + * is supported, but as the location isn't passed in, relative + * URIs are out. + * @param in instream + * @return a document + * @throws ParserConfigurationException parser feature problems + * @throws IOException IO problems + * @throws SAXException XML is invalid + */ + public static Document parseConfigXML(InputStream in) throws + ParserConfigurationException, + IOException, + SAXException { + DocumentBuilderFactory docBuilderFactory + = DocumentBuilderFactory.newInstance(); + //ignore all comments inside the xml file + docBuilderFactory.setIgnoringComments(true); + + //allow includes in the xml file + docBuilderFactory.setNamespaceAware(true); + docBuilderFactory.setXIncludeAware(true); + DocumentBuilder builder = docBuilderFactory.newDocumentBuilder(); + return builder.parse(in); + } + + /** + * Load a Hadoop configuration from a local file. + * @param file file to load + * @return a configuration which hasn't actually had the load triggered + * yet. + * @throws FileNotFoundException file is not there + * @throws IOException any other IO problem + */ + public static Configuration loadConfFromFile(File file) throws + IOException { + return loadConfFromFile(file, false); + } + + /** + * + * Load a Hadoop configuration from a local file. + * @param file file to load + * @param loadDefaults flag to indicate if the defaults should be loaded yet + * @return a configuration which hasn't actually had the load triggered + * yet. + * @throws FileNotFoundException file is not there + * @throws IOException any other IO problem + */ + public static Configuration loadConfFromFile(File file, + boolean loadDefaults) throws IOException { + if (!file.exists()) { + throw new FileNotFoundException("File not found :" + + file.getAbsoluteFile()); + } + Configuration conf = new Configuration(loadDefaults); + try { + conf.addResource(file.toURI().toURL()); + } catch (MalformedURLException e) { + // should never happen... + throw new IOException( + "File " + file.toURI() + " doesn't have a valid URL"); + } + return conf; + } + + /** + * Add a configuration from a file to an existing configuration + * @param conf existing configuration + * @param file file to load + * @param overwrite flag to indicate new values should overwrite the predecessor + * @return the merged configuration + * @throws IOException + */ + public static Configuration addConfigurationFile(Configuration conf, + File file, boolean overwrite) + throws IOException { + Configuration c2 = loadConfFromFile(file, false); + mergeConfigurations(conf, c2, file.getAbsolutePath(), overwrite); + return conf; + } + + /** + * Add the system env variables with the given prefix (by convention, env.) + * @param conf existing configuration + * @param prefix prefix + */ + public static void addEnvironmentVariables(Configuration conf, String prefix) { + Map<String, String> env = System.getenv(); + for (Map.Entry<String, String> entry : env.entrySet()) { + conf.set(prefix + entry.getKey(),entry.getValue(), "env"); + } + } + + /** + * looks for the config under $confdir/$templateFilename; if not there + * loads it from /conf/templateFile. + * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the + * origin to help debug what's happening + * @param systemConf system conf + * @param confdir conf dir in FS + * @param templateFilename filename in the confdir + * @param fallbackResource resource to fall back on + * @return loaded conf + * @throws IOException IO problems + */ + public static Configuration loadTemplateConfiguration(Configuration systemConf, + Path confdir, + String templateFilename, + String fallbackResource) throws + IOException { + FileSystem fs = FileSystem.get(confdir.toUri(), systemConf); + + Path templatePath = new Path(confdir, templateFilename); + return loadTemplateConfiguration(fs, templatePath, fallbackResource); + } + + /** + * looks for the config under $confdir/$templateFilename; if not there + * loads it from /conf/templateFile. + * The property {@link SliderKeys#KEY_TEMPLATE_ORIGIN} is set to the + * origin to help debug what's happening. + * @param fs Filesystem + * @param templatePath HDFS path for template + * @param fallbackResource resource to fall back on, or "" for no fallback + * @return loaded conf + * @throws IOException IO problems + * @throws FileNotFoundException if the path doesn't have a file and there + * was no fallback. + */ + public static Configuration loadTemplateConfiguration(FileSystem fs, + Path templatePath, + String fallbackResource) + throws IOException { + Configuration conf; + String origin; + if (fs.exists(templatePath)) { + log.debug("Loading template configuration {}", templatePath); + conf = loadConfiguration(fs, templatePath); + origin = templatePath.toString(); + } else { + if (fallbackResource.isEmpty()) { + throw new FileNotFoundException("No config file found at " + templatePath); + } + log.debug("Template {} not found" + + " -reverting to classpath resource {}", templatePath, fallbackResource); + conf = new Configuration(false); + conf.addResource(fallbackResource); + origin = "Resource " + fallbackResource; + } + //force a get + conf.get(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN); + //now set the origin + conf.set(SliderXmlConfKeys.KEY_TEMPLATE_ORIGIN, origin); + return conf; + } + + + /** + * For testing: dump a configuration + * @param conf configuration + * @return listing in key=value style + */ + public static String dumpConfigToString(Configuration conf) { + Set<String> sorted = sortedConfigKeys(conf); + + StringBuilder builder = new StringBuilder(); + for (String key : sorted) { + + builder.append(key) + .append("=") + .append(conf.get(key)) + .append("\n"); + } + return builder.toString(); + } + + /** + * Merge in one configuration above another + * @param base base config + * @param merge one to merge. This MUST be a non-default-load config to avoid + * merge origin confusion + * @param origin description of the origin for the put operation + * @param overwrite flag to indicate new values should overwrite the predecessor + * @return the base with the merged values + */ + public static Configuration mergeConfigurations(Configuration base, + Iterable<Map.Entry<String, String>> merge, + String origin, + boolean overwrite) { + for (Map.Entry<String, String> entry : merge) { + String key = entry.getKey(); + if (overwrite || base.get(key) == null) { + base.set(key, entry.getValue(), origin); + } + } + return base; + } + + /** + * Register a resource as a default resource. + * Do not attempt to use this unless you understand that the + * order in which default resources are loaded affects the outcome, + * and that subclasses of Configuration often register new default + * resources + * @param resource the resource name + * @return the URL or null + */ + public static URL registerDefaultResource(String resource) { + URL resURL = getResourceUrl(resource); + if (resURL != null) { + Configuration.addDefaultResource(resource); + } + return resURL; + } + + /** + * Load a configuration from a resource on this classpath. + * If the resource is not found, an empty configuration is returned + * @param resource the resource name + * @return the loaded configuration. + */ + public static Configuration loadFromResource(String resource) { + Configuration conf = new Configuration(false); + URL resURL = getResourceUrl(resource); + if (resURL != null) { + log.debug("loaded resources from {}", resURL); + conf.addResource(resource); + } else{ + log.debug("failed to find {} on the classpath", resource); + } + return conf; + + } + + /** + * Get the URL to a resource, null if not on the CP + * @param resource resource to look for + * @return the URL or null + */ + public static URL getResourceUrl(String resource) { + return ConfigHelper.class.getClassLoader() + .getResource(resource); + } + + /** + * Load a resource that must be on the classpath + * @param resource the resource name + * @return the loaded configuration + * @throws FileNotFoundException if the resource is missing + */ + public static Configuration loadMandatoryResource(String resource) + throws FileNotFoundException { + Configuration conf = new Configuration(false); + URL resURL = getResourceUrl(resource); + if (resURL != null) { + log.debug("loaded resources from {}", resURL); + conf.addResource(resource); + } else { + throw new FileNotFoundException(resource); + } + return conf; + } + + /** + * Propagate a property from a source to a dest config, with a best-effort + * attempt at propagating the origin. + * If the + * @param dest destination + * @param src source + * @param key key to try to copy + * @return true if the key was found and propagated + */ + public static boolean propagate(Configuration dest, + Configuration src, + String key) { + String val = src.get(key); + if (val != null) { + String[] origin = src.getPropertySources(key); + if (origin != null && origin.length > 0) { + dest.set(key, val, origin[0]); + } else { + dest.set(key, val); + return true; + } + } + return false; + } + + + /** + * Take a configuration, return a hash map + * @param conf conf + * @return hash map + */ + public static Map<String, String> buildMapFromConfiguration(Configuration conf) { + Map<String, String> map = new HashMap<String, String>(); + return SliderUtils.mergeEntries(map, conf); + } + + /** + * This goes through the keyset of one configuration and retrieves each value + * from a value source -a different or the same configuration. This triggers + * the property resolution process of the value, resolving any variables against + * in-config or inherited configurations + * @param keysource source of keys + * @param valuesource the source of values + * @return a new configuration where <code>foreach key in keysource, get(key)==valuesource.get(key)</code> + */ + public static Configuration resolveConfiguration( + Iterable<Map.Entry<String, String>> keysource, + Configuration valuesource) { + Configuration result = new Configuration(false); + for (Map.Entry<String, String> entry : keysource) { + String key = entry.getKey(); + String value = valuesource.get(key); + Preconditions.checkState(value != null, + "no reference for \"%s\" in values", key); + result.set(key, value); + } + return result; + } + + /** + * Register anything we consider deprecated + */ + public static void registerDeprecatedConfigItems() { + Configuration.addDeprecation( + SliderXmlConfKeys.REGISTRY_ZK_QUORUM, + RegistryConstants.KEY_REGISTRY_ZK_QUORUM); + Configuration.addDeprecation( + SliderXmlConfKeys.REGISTRY_PATH, + RegistryConstants.KEY_REGISTRY_ZK_ROOT); + + } + + /** + * Load a configuration with the {@link SliderKeys#SLIDER_XML} resource + * included + * @return a configuration instance + */ + public static Configuration loadSliderConfiguration() { + Configuration conf = new Configuration(); + conf.addResource(SliderKeys.SLIDER_XML); + return conf; + } + + /** + * Inject the {@link SliderKeys#SLIDER_XML} resource + * into the configuration resources <i>of all configurations</i>. + * <p> + * This operation is idempotent. + * <p> + * If the resource is not on the classpath, downgrades, rather than + * fails. + * @return true if the resource was found and loaded. + */ + public static synchronized boolean injectSliderXMLResource() { + if (sliderResourceInjectionAttempted.getAndSet(true)) { + return sliderResourceInjected.get(); + } + URL resourceUrl = getResourceUrl(SliderKeys.SLIDER_XML); + if (resourceUrl != null) { + Configuration.addDefaultResource(SliderKeys.SLIDER_XML); + sliderResourceInjected.set(true); + } + return sliderResourceInjected.get(); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/848f9490/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java new file mode 100644 index 0000000..6a02367 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java @@ -0,0 +1,831 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.tools; + +import com.google.common.base.Preconditions; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeys; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.yarn.api.records.LocalResource; +import org.apache.hadoop.yarn.api.records.LocalResourceType; +import org.apache.hadoop.yarn.api.records.LocalResourceVisibility; +import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.Records; +import org.apache.slider.common.SliderExitCodes; +import org.apache.slider.common.SliderKeys; +import org.apache.slider.common.SliderXmlConfKeys; +import org.apache.slider.core.exceptions.BadClusterStateException; +import org.apache.slider.core.exceptions.ErrorStrings; +import org.apache.slider.core.exceptions.SliderException; +import org.apache.slider.core.exceptions.UnknownApplicationInstanceException; +import org.apache.slider.core.persist.Filenames; +import org.apache.slider.core.persist.InstancePaths; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.HashMap; +import java.util.Map; +import java.util.Enumeration; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import static org.apache.slider.common.SliderXmlConfKeys.CLUSTER_DIRECTORY_PERMISSIONS; +import static org.apache.slider.common.SliderXmlConfKeys.DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS; + +public class CoreFileSystem { + private static final Logger + log = LoggerFactory.getLogger(CoreFileSystem.class); + + protected final FileSystem fileSystem; + protected final Configuration configuration; + + public CoreFileSystem(FileSystem fileSystem, Configuration configuration) { + Preconditions.checkNotNull(fileSystem, + "Cannot create a CoreFileSystem with a null FileSystem"); + Preconditions.checkNotNull(configuration, + "Cannot create a CoreFileSystem with a null Configuration"); + this.fileSystem = fileSystem; + this.configuration = configuration; + } + + public CoreFileSystem(Configuration configuration) throws IOException { + Preconditions.checkNotNull(configuration, + "Cannot create a CoreFileSystem with a null Configuration"); + this.fileSystem = FileSystem.get(configuration); + this.configuration = fileSystem.getConf(); + } + + /** + * Get the temp path for this cluster + * @param clustername name of the cluster + * @return path for temp files (is not purged) + */ + public Path getTempPathForCluster(String clustername) { + Path clusterDir = buildClusterDirPath(clustername); + return new Path(clusterDir, SliderKeys.TMP_DIR_PREFIX); + } + + /** + * Returns the underlying FileSystem for this object. + * + * @return filesystem + */ + public FileSystem getFileSystem() { + return fileSystem; + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("CoreFileSystem{"); + sb.append("fileSystem=").append(fileSystem.getUri()); + sb.append('}'); + return sb.toString(); + } + + /** + * Build up the path string for a cluster instance -no attempt to + * create the directory is made + * + * @param clustername name of the cluster + * @return the path for persistent data + */ + public Path buildClusterDirPath(String clustername) { + Preconditions.checkNotNull(clustername); + Path path = getBaseApplicationPath(); + return new Path(path, SliderKeys.CLUSTER_DIRECTORY + "/" + clustername); + } + + /** + * Build up the path string for app def folder -no attempt to + * create the directory is made + * + * @param clustername name of the cluster + * @return the path for persistent data + */ + public Path buildAppDefDirPath(String clustername) { + Path path = buildClusterDirPath(clustername); + return new Path(path, SliderKeys.APP_DEF_DIR); + } + + /** + * Build up the path string for addon folder -no attempt to + * create the directory is made + * + * @param clustername name of the cluster + * @return the path for persistent data + */ + public Path buildAddonDirPath(String clustername, String addonId) { + Preconditions.checkNotNull(addonId); + Path path = buildClusterDirPath(clustername); + return new Path(path, SliderKeys.ADDONS_DIR + "/" + addonId); + } + + /** + * Build up the path string for package install location -no attempt to + * create the directory is made + * + * @return the path for persistent app package + */ + public Path buildPackageDirPath(String packageName, String packageVersion) { + Preconditions.checkNotNull(packageName); + Path path = getBaseApplicationPath(); + path = new Path(path, SliderKeys.PACKAGE_DIRECTORY + "/" + packageName); + if (SliderUtils.isSet(packageVersion)) { + path = new Path(path, packageVersion); + } + return path; + } + + /** + * Build up the path string for package install location -no attempt to + * create the directory is made + * + * @return the path for persistent app package + */ + public Path buildClusterSecurityDirPath(String clusterName) { + Preconditions.checkNotNull(clusterName); + Path path = buildClusterDirPath(clusterName); + return new Path(path, SliderKeys.SECURITY_DIR); + } + + /** + * Build up the path string for keytab install location -no attempt to + * create the directory is made + * + * @return the path for keytab + */ + public Path buildKeytabInstallationDirPath(String keytabFolder) { + Preconditions.checkNotNull(keytabFolder); + Path path = getBaseApplicationPath(); + return new Path(path, SliderKeys.KEYTAB_DIR + "/" + keytabFolder); + } + + /** + * Build up the path string for keytab install location -no attempt to + * create the directory is made + * + * @return the path for keytab installation location + */ + public Path buildKeytabPath(String keytabDir, String keytabName, String clusterName) { + Path homePath = getHomeDirectory(); + Path baseKeytabDir; + if (keytabDir != null) { + baseKeytabDir = new Path(homePath, keytabDir); + } else { + baseKeytabDir = new Path(buildClusterDirPath(clusterName), + SliderKeys.KEYTAB_DIR); + } + return keytabName == null ? baseKeytabDir : + new Path(baseKeytabDir, keytabName); + } + + /** + * Create the Slider cluster path for a named cluster and all its subdirs + * This is a directory; a mkdirs() operation is executed + * to ensure that it is there. + * + * @param clustername name of the cluster + * @return the path to the cluster directory + * @throws java.io.IOException trouble + * @throws SliderException slider-specific exceptions + */ + public Path createClusterDirectories(String clustername, Configuration conf) + throws IOException, SliderException { + + + Path clusterDirectory = buildClusterDirPath(clustername); + InstancePaths instancePaths = new InstancePaths(clusterDirectory); + createClusterDirectories(instancePaths); + return clusterDirectory; + } + + /** + * Create the Slider cluster path for a named cluster and all its subdirs + * This is a directory; a mkdirs() operation is executed + * to ensure that it is there. + * + * @param instancePaths instance paths + * @throws IOException trouble + * @throws SliderException slider-specific exceptions + */ + public void createClusterDirectories(InstancePaths instancePaths) throws + IOException, SliderException { + Path instanceDir = instancePaths.instanceDir; + + verifyDirectoryNonexistent(instanceDir); + FsPermission clusterPerms = getInstanceDirectoryPermissions(); + createWithPermissions(instanceDir, clusterPerms); + createWithPermissions(instancePaths.snapshotConfPath, clusterPerms); + createWithPermissions(instancePaths.generatedConfPath, clusterPerms); + createWithPermissions(instancePaths.historyPath, clusterPerms); + createWithPermissions(instancePaths.tmpPathAM, clusterPerms); + + // Data Directory + String dataOpts = + configuration.get(SliderXmlConfKeys.DATA_DIRECTORY_PERMISSIONS, + SliderXmlConfKeys.DEFAULT_DATA_DIRECTORY_PERMISSIONS); + log.debug("Setting data directory permissions to {}", dataOpts); + createWithPermissions(instancePaths.dataPath, new FsPermission(dataOpts)); + + } + + /** + * Create a directory with the given permissions. + * + * @param dir directory + * @param clusterPerms cluster permissions + * @throws IOException IO problem + * @throws BadClusterStateException any cluster state problem + */ + public void createWithPermissions(Path dir, FsPermission clusterPerms) throws + IOException, + BadClusterStateException { + if (fileSystem.isFile(dir)) { + // HADOOP-9361 shows some filesystems don't correctly fail here + throw new BadClusterStateException( + "Cannot create a directory over a file %s", dir); + } + log.debug("mkdir {} with perms {}", dir, clusterPerms); + //no mask whatoever + fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "000"); + fileSystem.mkdirs(dir, clusterPerms); + //and force set it anyway just to make sure + fileSystem.setPermission(dir, clusterPerms); + } + + /** + * Get the permissions of a path + * + * @param path path to check + * @return the permissions + * @throws IOException any IO problem (including file not found) + */ + public FsPermission getPathPermissions(Path path) throws IOException { + FileStatus status = fileSystem.getFileStatus(path); + return status.getPermission(); + } + + public FsPermission getInstanceDirectoryPermissions() { + String clusterDirPermsOct = + configuration.get(CLUSTER_DIRECTORY_PERMISSIONS, + DEFAULT_CLUSTER_DIRECTORY_PERMISSIONS); + return new FsPermission(clusterDirPermsOct); + } + + /** + * Verify that the cluster directory is not present + * + * @param clustername name of the cluster + * @param clusterDirectory actual directory to look for + * @throws IOException trouble with FS + * @throws SliderException If the directory exists + */ + public void verifyClusterDirectoryNonexistent(String clustername, + Path clusterDirectory) + throws IOException, SliderException { + if (fileSystem.exists(clusterDirectory)) { + throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS, + ErrorStrings.PRINTF_E_INSTANCE_ALREADY_EXISTS, clustername, + clusterDirectory); + } + } + /** + * Verify that the given directory is not present + * + * @param clusterDirectory actual directory to look for + * @throws IOException trouble with FS + * @throws SliderException If the directory exists + */ + public void verifyDirectoryNonexistent(Path clusterDirectory) throws + IOException, + SliderException { + if (fileSystem.exists(clusterDirectory)) { + + log.error("Dir {} exists: {}", + clusterDirectory, + listFSDir(clusterDirectory)); + throw new SliderException(SliderExitCodes.EXIT_INSTANCE_EXISTS, + ErrorStrings.PRINTF_E_INSTANCE_DIR_ALREADY_EXISTS, + clusterDirectory); + } + } + + /** + * Verify that a user has write access to a directory. + * It does this by creating then deleting a temp file + * + * @param dirPath actual directory to look for + * @throws FileNotFoundException file not found + * @throws IOException trouble with FS + * @throws BadClusterStateException if the directory is not writeable + */ + public void verifyDirectoryWriteAccess(Path dirPath) throws IOException, + SliderException { + verifyPathExists(dirPath); + Path tempFile = new Path(dirPath, "tmp-file-for-checks"); + try { + FSDataOutputStream out ; + out = fileSystem.create(tempFile, true); + IOUtils.closeStream(out); + fileSystem.delete(tempFile, false); + } catch (IOException e) { + log.warn("Failed to create file {}: {}", tempFile, e); + throw new BadClusterStateException(e, + "Unable to write to directory %s : %s", dirPath, e.toString()); + } + } + + /** + * Verify that a path exists + * @param path path to check + * @throws FileNotFoundException file not found + * @throws IOException trouble with FS + */ + public void verifyPathExists(Path path) throws IOException { + if (!fileSystem.exists(path)) { + throw new FileNotFoundException(path.toString()); + } + } + + /** + * Verify that a path exists + * @param path path to check + * @throws FileNotFoundException file not found or is not a file + * @throws IOException trouble with FS + */ + public void verifyFileExists(Path path) throws IOException { + FileStatus status = fileSystem.getFileStatus(path); + + if (!status.isFile()) { + throw new FileNotFoundException("Not a file: " + path.toString()); + } + } + + /** + * Given a path, check if it exists and is a file + * + * @param path + * absolute path to the file to check + * @returns true if and only if path exists and is a file, false for all other + * reasons including if file check throws IOException + */ + public boolean isFile(Path path) { + boolean isFile = false; + try { + FileStatus status = fileSystem.getFileStatus(path); + if (status.isFile()) { + isFile = true; + } + } catch (IOException e) { + // ignore, isFile is already set to false + } + return isFile; + } + + /** + * Verify that a file exists in the zip file given by path + * @param path path to zip file + * @param file file expected to be in zip + * @throws FileNotFoundException file not found or is not a zip file + * @throws IOException trouble with FS + */ + public void verifyFileExistsInZip(Path path, String file) throws IOException { + fileSystem.copyToLocalFile(path, new Path("/tmp")); + File dst = new File((new Path("/tmp", path.getName())).toString()); + Enumeration<? extends ZipEntry> entries; + ZipFile zipFile = new ZipFile(dst); + boolean found = false; + + try { + entries = zipFile.entries(); + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + String nm = entry.getName(); + if (nm.endsWith(file)) { + found = true; + break; + } + } + } finally { + zipFile.close(); + } + dst.delete(); + if (!found) throw new FileNotFoundException("file: " + file + " not found in " + path); + log.info("Verification of " + path + " passed"); + } + /** + * Create the application-instance specific temporary directory + * in the DFS + * + * @param clustername name of the cluster + * @param subdir application ID + * @return the path; this directory will already have been created + */ + public Path createAppInstanceTempPath(String clustername, String subdir) + throws IOException { + Path tmp = getTempPathForCluster(clustername); + Path instancePath = new Path(tmp, subdir); + fileSystem.mkdirs(instancePath); + return instancePath; + } + + /** + * Create the application-instance specific temporary directory + * in the DFS + * + * @param clustername name of the cluster + * @return the path; this directory will already have been deleted + */ + public Path purgeAppInstanceTempFiles(String clustername) throws + IOException { + Path tmp = getTempPathForCluster(clustername); + fileSystem.delete(tmp, true); + return tmp; + } + + /** + * Get the base path + * + * @return the base path optionally configured by + * {@link SliderXmlConfKeys#KEY_SLIDER_BASE_PATH} + */ + public Path getBaseApplicationPath() { + String configuredBasePath = configuration.get(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH); + return configuredBasePath != null ? new Path(configuredBasePath) : + new Path(getHomeDirectory(), SliderKeys.SLIDER_BASE_DIRECTORY); + } + + /** + * Get slider dependency parent dir in HDFS + * + * @return the parent dir path of slider.tar.gz in HDFS + */ + public Path getDependencyPath() { + String parentDir = (SliderUtils.isHdp()) ? SliderKeys.SLIDER_DEPENDENCY_HDP_PARENT_DIR + + SliderKeys.SLIDER_DEPENDENCY_DIR + : SliderKeys.SLIDER_DEPENDENCY_DIR; + Path dependencyPath = new Path(String.format(parentDir, + SliderUtils.getSliderVersion())); + return dependencyPath; + } + + /** + * Get slider.tar.gz absolute filepath in HDFS + * + * @return the absolute path to slider.tar.gz in HDFS + */ + public Path getDependencyTarGzip() { + Path dependencyLibAmPath = getDependencyPath(); + Path dependencyLibTarGzip = new Path( + dependencyLibAmPath.toUri().toString(), + SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_NAME + + SliderKeys.SLIDER_DEPENDENCY_TAR_GZ_FILE_EXT); + return dependencyLibTarGzip; + } + + public Path getHomeDirectory() { + return fileSystem.getHomeDirectory(); + } + + public boolean maybeAddImagePath(Map<String, LocalResource> localResources, + Path imagePath) throws IOException { + if (imagePath != null) { + LocalResource resource = createAmResource(imagePath, + LocalResourceType.ARCHIVE); + localResources.put(SliderKeys.LOCAL_TARBALL_INSTALL_SUBDIR, resource); + return true; + } else { + return false; + } + } + + public boolean maybeAddImagePath(Map<String, LocalResource> localResources, + String imagePath) throws IOException { + + return imagePath != null && + maybeAddImagePath(localResources, new Path(imagePath)); + } + + + + + /** + * Create an AM resource from the + * + * @param destPath dest path in filesystem + * @param resourceType resource type + * @return the resource set up wih application-level visibility and the + * timestamp & size set from the file stats. + */ + public LocalResource createAmResource(Path destPath, LocalResourceType resourceType) throws IOException { + FileStatus destStatus = fileSystem.getFileStatus(destPath); + LocalResource amResource = Records.newRecord(LocalResource.class); + amResource.setType(resourceType); + // Set visibility of the resource + // Setting to most private option + amResource.setVisibility(LocalResourceVisibility.APPLICATION); + // Set the resource to be copied over + amResource.setResource(ConverterUtils.getYarnUrlFromPath(fileSystem + .resolvePath(destStatus.getPath()))); + // Set timestamp and length of file so that the framework + // can do basic sanity checks for the local resource + // after it has been copied over to ensure it is the same + // resource the client intended to use with the application + amResource.setTimestamp(destStatus.getModificationTime()); + amResource.setSize(destStatus.getLen()); + return amResource; + } + + /** + * Register all files under a fs path as a directory to push out + * + * @param srcDir src dir + * @param destRelativeDir dest dir (no trailing /) + * @return the map of entries + */ + public Map<String, LocalResource> submitDirectory(Path srcDir, String destRelativeDir) throws IOException { + //now register each of the files in the directory to be + //copied to the destination + FileStatus[] fileset = fileSystem.listStatus(srcDir); + Map<String, LocalResource> localResources = + new HashMap<String, LocalResource>(fileset.length); + for (FileStatus entry : fileset) { + + LocalResource resource = createAmResource(entry.getPath(), + LocalResourceType.FILE); + String relativePath = destRelativeDir + "/" + entry.getPath().getName(); + localResources.put(relativePath, resource); + } + return localResources; + } + + /** + * Submit a JAR containing a specific class, returning + * the resource to be mapped in + * + * @param clazz class to look for + * @param subdir subdirectory (expected to end in a "/") + * @param jarName <i>At the destination</i> + * @return the local resource ref + * @throws IOException trouble copying to HDFS + */ + public LocalResource submitJarWithClass(Class clazz, Path tempPath, String subdir, String jarName) + throws IOException, SliderException { + File localFile = SliderUtils.findContainingJarOrFail(clazz); + return submitFile(localFile, tempPath, subdir, jarName); + } + + /** + * Submit a local file to the filesystem references by the instance's cluster + * filesystem + * + * @param localFile filename + * @param subdir subdirectory (expected to end in a "/") + * @param destFileName destination filename + * @return the local resource ref + * @throws IOException trouble copying to HDFS + */ + public LocalResource submitFile(File localFile, Path tempPath, String subdir, String destFileName) + throws IOException { + Path src = new Path(localFile.toString()); + Path subdirPath = new Path(tempPath, subdir); + fileSystem.mkdirs(subdirPath); + Path destPath = new Path(subdirPath, destFileName); + log.debug("Copying {} (size={} bytes) to {}", localFile, localFile.length(), destPath); + + fileSystem.copyFromLocalFile(false, true, src, destPath); + + // Set the type of resource - file or archive + // archives are untarred at destination + // we don't need the jar file to be untarred for now + return createAmResource(destPath, LocalResourceType.FILE); + } + + /** + * Submit the AM tar.gz resource referenced by the instance's cluster + * filesystem. Also, update the providerResources object with the new + * resource. + * + * @param providerResources + * the provider resource map to be updated + * @throws IOException + * trouble copying to HDFS + */ + public void submitTarGzipAndUpdate( + Map<String, LocalResource> providerResources) throws IOException, + BadClusterStateException { + Path dependencyLibTarGzip = getDependencyTarGzip(); + LocalResource lc = createAmResource(dependencyLibTarGzip, + LocalResourceType.ARCHIVE); + providerResources.put(SliderKeys.SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK, lc); + } + + /** + * Copy local file(s) to destination HDFS directory. If {@code localPath} is a + * local directory then all files matching the {@code filenameFilter} + * (optional) are copied, otherwise {@code filenameFilter} is ignored. + * + * @param localPath + * a local file or directory path + * @param filenameFilter + * if {@code localPath} is a directory then filenameFilter is used as + * a filter (if specified) + * @param destDir + * the destination HDFS directory where the file(s) should be copied + * @param fp + * file permissions of all the directories and files that will be + * created in this api + * @throws IOException + */ + public void copyLocalFilesToHdfs(File localPath, + FilenameFilter filenameFilter, Path destDir, FsPermission fp) + throws IOException { + if (localPath == null || destDir == null) { + throw new IOException("Either localPath or destDir is null"); + } + fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, + "000"); + fileSystem.mkdirs(destDir, fp); + if (localPath.isDirectory()) { + // copy all local files under localPath to destDir (honoring filename + // filter if provided + File[] localFiles = localPath.listFiles(filenameFilter); + Path[] localFilePaths = new Path[localFiles.length]; + int i = 0; + for (File localFile : localFiles) { + localFilePaths[i++] = new Path(localFile.getPath()); + } + log.info("Copying {} files from {} to {}", i, localPath.toURI(), + destDir.toUri()); + fileSystem.copyFromLocalFile(false, true, localFilePaths, destDir); + } else { + log.info("Copying file {} to {}", localPath.toURI(), destDir.toUri()); + fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()), + destDir); + } + // set permissions for all the files created in the destDir + fileSystem.setPermission(destDir, fp); + } + + public void copyLocalFileToHdfs(File localPath, + Path destPath, FsPermission fp) + throws IOException { + if (localPath == null || destPath == null) { + throw new IOException("Either localPath or destPath is null"); + } + fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, + "000"); + fileSystem.mkdirs(destPath.getParent(), fp); + log.info("Copying file {} to {}", localPath.toURI(), destPath.toUri()); + + fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()), + destPath); + // set file permissions of the destPath + fileSystem.setPermission(destPath, fp); + } + + /** + * list entries in a filesystem directory + * + * @param path directory + * @return a listing, one to a line + * @throws IOException + */ + public String listFSDir(Path path) throws IOException { + FileStatus[] stats = fileSystem.listStatus(path); + StringBuilder builder = new StringBuilder(); + for (FileStatus stat : stats) { + builder.append(stat.getPath().toString()) + .append("\t") + .append(stat.getLen()) + .append("\n"); + } + return builder.toString(); + } + + /** + * List all application instances persisted for this user, giving the + * path. The instance name is the last element in the path + * @return a possibly empty map of application instance names to paths + */ + public Map<String, Path> listPersistentInstances() throws IOException { + FileSystem fs = getFileSystem(); + Path path = new Path(getBaseApplicationPath(), SliderKeys.CLUSTER_DIRECTORY); + log.debug("Looking for all persisted application at {}", path.toString()); + if (!fs.exists(path)) { + // special case: no instances have ever been created + return new HashMap<String, Path>(0); + } + FileStatus[] statuses = fs.listStatus(path); + Map<String, Path> instances = new HashMap<String, Path>(statuses.length); + + // enum the child entries + for (FileStatus status : statuses) { + if (status.isDirectory()) { + // for directories, look for an internal.json underneath + Path child = status.getPath(); + Path internalJson = new Path(child, Filenames.INTERNAL); + if (fs.exists(internalJson)) { + // success => this is an instance + instances.put(child.getName(), child); + } else { + log.info("Malformed cluster found at {}. It does not appear to be a valid persisted instance.", + child.toString()); + } + } + } + return instances; + } + + public void touch(Path path, boolean overwrite) throws IOException { + FSDataOutputStream out = fileSystem.create(path, overwrite); + out.close(); + } + + public void cat(Path path, boolean overwrite, String data) throws IOException { + FSDataOutputStream out = fileSystem.create(path, overwrite); + byte[] bytes = data.getBytes(Charset.forName("UTF-8")); + out.write(bytes); + out.close(); + } + + /** + * Create a path that must exist in the cluster fs + * @param uri uri to create + * @return the path + * @throws SliderException if the path does not exist + */ + public Path createPathThatMustExist(String uri) throws + SliderException, IOException { + Preconditions.checkNotNull(uri); + Path path = new Path(uri); + verifyPathExists(path); + return path; + } + + /** + * Locate an application conf json in the FS. This includes a check to verify + * that the file is there. + * + * @param clustername name of the cluster + * @return the path to the spec. + * @throws IOException IO problems + * @throws SliderException if the path isn't there + */ + public Path locateInstanceDefinition(String clustername) throws IOException, + SliderException { + Path clusterDirectory = buildClusterDirPath(clustername); + Path appConfPath = + new Path(clusterDirectory, Filenames.APPCONF); + verifyClusterSpecExists(clustername, appConfPath); + return appConfPath; + } + + /** + * Verify that a cluster specification exists + * @param clustername name of the cluster (For errors only) + * @param clusterSpecPath cluster specification path + * @throws IOException IO problems + * @throws SliderException if the cluster specification is not present + */ + public void verifyClusterSpecExists(String clustername, Path clusterSpecPath) + throws IOException, + SliderException { + if (!fileSystem.isFile(clusterSpecPath)) { + log.debug("Missing specification file {}", clusterSpecPath); + throw UnknownApplicationInstanceException.unknownInstance( + clustername + "\n (definition not found at " + clusterSpecPath); + } + } + + +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
