http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java index 59ab30b..6df4bf4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ContainerOutcome.java @@ -29,7 +29,7 @@ public enum ContainerOutcome { Completed, Failed, Failed_limits_exceeded, - Node_failure, + Disk_failure, Preempted; /** @@ -48,7 +48,7 @@ public enum ContainerOutcome { // could either be a release or node failure. Treat as completion return Completed; case ContainerExitStatus.DISKS_FAILED: - return Node_failure; + return Disk_failure; case ContainerExitStatus.PREEMPTED: return Preempted; case ContainerExitStatus.KILLED_EXCEEDED_PMEM:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java index eb8ff03..d57b6d2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/NodeEntry.java @@ -222,7 +222,7 @@ public class NodeEntry implements Cloneable { // general "any reason" app failure case Failed: // specific node failure - case Node_failure: + case Disk_failure: ++failed; ++failedRecently; http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java index 8e8546b..5051aee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java @@ -21,7 +21,6 @@ package org.apache.slider.server.appmaster.state; import com.codahale.metrics.Metric; import com.codahale.metrics.MetricSet; import com.google.common.base.Preconditions; -import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.slider.api.types.ComponentInformation; import org.apache.slider.api.types.RoleStatistics; @@ -35,8 +34,6 @@ import java.util.Comparator; import java.util.HashMap; import java.util.Map; -import static org.apache.hadoop.metrics2.lib.Interns.info; - /** * Models the ongoing status of all nodes in an application. * @@ -207,12 +204,16 @@ public final class RoleStatus implements MetricSet { return componentMetrics.containersDesired.value(); } - long getRunning() { + public void setDesired(int desired) { + componentMetrics.containersDesired.set(desired); + } + + public long getRunning() { return componentMetrics.containersRunning.value(); } - public long getPending() { - return componentMetrics.containersPending.value(); + public long getRequested() { + return componentMetrics.containersRequested.value(); } public long getAAPending() { @@ -222,22 +223,35 @@ public final class RoleStatus implements MetricSet { void decAAPending() { componentMetrics.pendingAAContainers.decr(); } + void setAAPending(long n) { componentMetrics.pendingAAContainers.set((int)n); } - long getFailedRecently() { + public long getLimitsExceeded() { + return componentMetrics.containersLimitsExceeded.value(); + } + + public long getPreempted() { + return componentMetrics.containersPreempted.value(); + } + + public long getDiskFailed() { + return componentMetrics.containersDiskFailure.value(); + } + + public long getFailedRecently() { return componentMetrics.failedSinceLastThreshold.value(); } - long resetFailedRecently() { + public long resetFailedRecently() { long count = componentMetrics.failedSinceLastThreshold.value(); componentMetrics.failedSinceLastThreshold.set(0); return count; } - long getFailed() { + public long getFailed() { return componentMetrics.containersFailed.value(); } @@ -254,6 +268,8 @@ public final class RoleStatus implements MetricSet { long inuse = getActualAndRequested(); long delta = getDesired() - inuse; if (delta < 0) { + // TODO this doesn't do anything now that we're not tracking releasing + // containers -- maybe we need releasing //if we are releasing, remove the number that are already released. //but never switch to a positive delta = Math.min(delta, 0); @@ -262,11 +278,11 @@ public final class RoleStatus implements MetricSet { } /** - * Get count of actual and requested containers. This includes pending ones + * Get count of actual and requested containers. * @return the size of the application when outstanding requests are included. */ public long getActualAndRequested() { - return getRunning() + getPending(); + return getRunning() + getRequested(); } /** @@ -341,6 +357,14 @@ public final class RoleStatus implements MetricSet { public synchronized RoleStatistics getStatistics() { RoleStatistics stats = new RoleStatistics(); stats.activeAA = getOutstandingAARequestCount(); + stats.actual = getRunning(); + stats.desired = getDesired(); + stats.failed = getFailed(); + stats.limitsExceeded = getLimitsExceeded(); + stats.nodeFailed = getDiskFailed(); + stats.preempted = getPreempted(); + stats.requested = getRequested(); + stats.started = getRunning(); return stats; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java index 440094e..8dca4ed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/web/view/IndexBlock.java @@ -156,7 +156,7 @@ public class IndexBlock extends SliderHamletBlock { } else { aatext = ""; } - if (status.getPending() > 0) { + if (status.getRequested() > 0) { roleWithOpenRequest ++; } } @@ -165,7 +165,7 @@ public class IndexBlock extends SliderHamletBlock { .td().a(nameUrl, roleName)._() .td(String.format("%d", metrics.containersDesired.value())) .td(String.format("%d", metrics.containersRunning.value())) - .td(String.format("%d", metrics.containersPending.value())) + .td(String.format("%d", metrics.containersRequested.value())) .td(String.format("%d", metrics.containersFailed.value())) .td(aatext) ._(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto index bfcab23..691f861 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterMessages.proto @@ -80,12 +80,16 @@ message UpgradeContainersRequestProto { message UpgradeContainersResponseProto { } -message FlexComponentRequestProto { +message FlexComponentsRequestProto { + repeated ComponentCountProto components = 1; +} + +message ComponentCountProto { optional string name = 1; - optional int32 numberOfContainers = 2; + optional int64 numberOfContainers = 2; } -message FlexComponentResponseProto { +message FlexComponentsResponseProto { } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto index f52d7a1..776ce28 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/proto/SliderClusterProtocol.proto @@ -61,7 +61,7 @@ service SliderClusterProtocolPB { rpc upgradeContainers(UpgradeContainersRequestProto) returns(UpgradeContainersResponseProto); - rpc flexComponent(FlexComponentRequestProto) returns (FlexComponentResponseProto); + rpc flexComponents(FlexComponentsRequestProto) returns (FlexComponentsResponseProto); /** * Get the current cluster status http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/api/TestRPCBinding.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/api/TestRPCBinding.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/api/TestRPCBinding.java new file mode 100644 index 0000000..28483dc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/api/TestRPCBinding.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api; + +import org.apache.hadoop.conf.Configuration; +import org.apache.slider.server.appmaster.rpc.RpcBinder; +import org.apache.slider.server.appmaster.rpc.SliderClusterProtocolPB; +import org.junit.Test; + +import java.net.InetSocketAddress; + +import static org.junit.Assert.assertTrue; + +/** + * Tests RPC work. + */ +public class TestRPCBinding { + + @Test + public void testRegistration() throws Throwable { + Configuration conf = new Configuration(); + RpcBinder.registerSliderAPI(conf); + assertTrue(RpcBinder.verifyBondedToProtobuf(conf, + SliderClusterProtocolPB.class)); + } + + @Test + public void testGetProxy() throws Throwable { + Configuration conf = new Configuration(); + InetSocketAddress saddr = new InetSocketAddress("127.0.0.1", 9000); + SliderClusterProtocol proxy = + RpcBinder.connectToServer(saddr, null, conf, 1000); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java new file mode 100644 index 0000000..6299a9c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBadArgs.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.client; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.slider.common.params.Arguments; +import org.apache.slider.common.params.SliderActions; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; +import org.apache.slider.core.exceptions.ErrorStrings; +import org.apache.slider.core.exceptions.UsageException; +import org.apache.slider.utils.SliderTestBase; +import org.junit.Ignore; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; + +/** + * Test the argument parsing/validation logic. + */ +public class TestClientBadArgs extends SliderTestBase { + private static final Logger LOG = + LoggerFactory.getLogger(TestClientBadArgs.class); + + @Test + public void testNoAction() throws Throwable { + launchExpectingException(SliderClient.class, + createTestConfig(), + "Usage: slider COMMAND", + EMPTY_LIST); + + } + + @Test + public void testUnknownAction() throws Throwable { + launchExpectingException(SliderClient.class, + createTestConfig(), + "not-a-known-action", + Arrays.asList("not-a-known-action")); + } + + @Test + public void testActionWithoutOptions() throws Throwable { + launchExpectingException(SliderClient.class, + createTestConfig(), + "Usage: slider build <application>", + Arrays.asList(SliderActions.ACTION_BUILD)); + } + + @Test + public void testActionWithoutEnoughArgs() throws Throwable { + launchExpectingException(SliderClient.class, + createTestConfig(), + ErrorStrings.ERROR_NOT_ENOUGH_ARGUMENTS, + Arrays.asList(SliderActions.ACTION_START)); + } + + @Test + public void testActionWithTooManyArgs() throws Throwable { + launchExpectingException(SliderClient.class, + createTestConfig(), + ErrorStrings.ERROR_TOO_MANY_ARGUMENTS, + Arrays.asList(SliderActions.ACTION_HELP, + "hello, world")); + } + + @Test + public void testBadImageArg() throws Throwable { + launchExpectingException(SliderClient.class, + createTestConfig(), + "Unknown option: --image", + Arrays.asList(SliderActions.ACTION_HELP, + Arguments.ARG_IMAGE)); + } + + @Test + public void testRegistryUsage() throws Throwable { + Throwable exception = launchExpectingException(SliderClient.class, + createTestConfig(), + "org.apache.slider.core.exceptions.UsageException: Argument --name " + + "missing", + Arrays.asList(SliderActions.ACTION_REGISTRY)); + assertTrue(exception instanceof UsageException); + LOG.info(exception.toString()); + } + + @Test + public void testRegistryExportBadUsage1() throws Throwable { + Throwable exception = launchExpectingException(SliderClient.class, + createTestConfig(), + "Expected a value after parameter --getexp", + Arrays.asList(SliderActions.ACTION_REGISTRY, + Arguments.ARG_NAME, + "cl1", + Arguments.ARG_GETEXP)); + assertTrue(exception instanceof BadCommandArgumentsException); + LOG.info(exception.toString()); + } + + @Test + public void testRegistryExportBadUsage2() throws Throwable { + Throwable exception = launchExpectingException(SliderClient.class, + createTestConfig(), + "Expected a value after parameter --getexp", + Arrays.asList(SliderActions.ACTION_REGISTRY, + Arguments.ARG_NAME, + "cl1", + Arguments.ARG_LISTEXP, + Arguments.ARG_GETEXP)); + assertTrue(exception instanceof BadCommandArgumentsException); + LOG.info(exception.toString()); + } + + @Test + public void testRegistryExportBadUsage3() throws Throwable { + Throwable exception = launchExpectingException(SliderClient.class, + createTestConfig(), + "Usage: registry", + Arrays.asList(SliderActions.ACTION_REGISTRY, + Arguments.ARG_NAME, + "cl1", + Arguments.ARG_LISTEXP, + Arguments.ARG_GETEXP, + "export1")); + assertTrue(exception instanceof UsageException); + LOG.info(exception.toString()); + } + + @Test + public void testUpgradeUsage() throws Throwable { + Throwable exception = launchExpectingException(SliderClient.class, + createTestConfig(), + "org.apache.slider.core.exceptions.BadCommandArgumentsException: Not " + + "enough arguments for action: upgrade Expected minimum 1 but got 0", + Arrays.asList(SliderActions.ACTION_UPGRADE)); + assertTrue(exception instanceof BadCommandArgumentsException); + LOG.info(exception.toString()); + } + + public Configuration createTestConfig() { + Configuration configuration = new Configuration(); + configuration.set(YarnConfiguration.RM_ADDRESS, "127.0.0.1:8032"); + return configuration; + } + + @Ignore + @Test + public void testUpgradeWithTemplateResourcesAndContainersOption() throws + Throwable { + //TODO test upgrade args + String appName = "test_hbase"; + Throwable exception = launchExpectingException(SliderClient.class, + createTestConfig(), + "BadCommandArgumentsException: Option --containers cannot be " + + "specified with --appdef", + Arrays.asList(SliderActions.ACTION_UPGRADE, + appName, + Arguments.ARG_APPDEF, + "/tmp/app.json", + Arguments.ARG_CONTAINERS, + "container_1" + )); + assertTrue(exception instanceof BadCommandArgumentsException); + LOG.info(exception.toString()); + } + + @Ignore + @Test + public void testUpgradeWithTemplateResourcesAndComponentsOption() throws + Throwable { + //TODO test upgrade args + String appName = "test_hbase"; + Throwable exception = launchExpectingException(SliderClient.class, + createTestConfig(), + "BadCommandArgumentsException: Option --components cannot be " + + "specified with --appdef", + Arrays.asList(SliderActions.ACTION_UPGRADE, + appName, + Arguments.ARG_APPDEF, + "/tmp/app.json", + Arguments.ARG_COMPONENTS, + "HBASE_MASTER" + )); + assertTrue(exception instanceof BadCommandArgumentsException); + LOG.info(exception.toString()); + } + + @Test + public void testNodesMissingFile() throws Throwable { + Throwable exception = launchExpectingException(SliderClient.class, + createTestConfig(), + "after parameter --out", + Arrays.asList(SliderActions.ACTION_NODES, Arguments.ARG_OUTPUT)); + assertTrue(exception instanceof BadCommandArgumentsException); + } + + @Test + public void testFlexWithNoComponents() throws Throwable { + Throwable exception = launchExpectingException(SliderClient.class, + new Configuration(), + "Usage: slider flex <application>", + Arrays.asList( + SliderActions.ACTION_FLEX, + "flex1", + Arguments.ARG_DEFINE, + YarnConfiguration.RM_ADDRESS + "=127.0.0.1:8032" + )); + assertTrue(exception instanceof UsageException); + LOG.info(exception.toString()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBasicArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBasicArgs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBasicArgs.java new file mode 100644 index 0000000..43c5163 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestClientBasicArgs.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.client; + +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.slider.common.params.Arguments; +import org.apache.slider.common.params.ClientArgs; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.main.ServiceLauncher; +import org.apache.slider.utils.SliderTestBase; +import org.junit.Test; + +import java.net.UnknownHostException; +import java.util.Arrays; + +/** + * Test bad argument handling. + */ +public class TestClientBasicArgs extends SliderTestBase { + + /** + * Help should print out help string and then succeed. + * @throws Throwable + */ + @Test + public void testHelp() throws Throwable { + ServiceLauncher launcher = launch(SliderClient.class, + SliderUtils.createConfiguration(), + Arrays.asList(ClientArgs.ACTION_HELP)); + assertEquals(0, launcher.getServiceExitCode()); + } + + @Test + public void testNoArgs() throws Throwable { + launchExpectingException(SliderClient.class, + SliderUtils.createConfiguration(), + "Usage: slider COMMAND", + EMPTY_LIST); + } + + @Test + public void testListUnknownRM() throws Throwable { + try { + YarnConfiguration conf = SliderUtils.createConfiguration(); + conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, + 1000); + conf.setLong(YarnConfiguration + .RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1000); + ServiceLauncher launcher = launch(SliderClient.class, + conf, + Arrays.asList( + ClientArgs.ACTION_LIST, + "cluster", + Arguments.ARG_MANAGER, + "badhost:8888")); + fail("expected an exception, got a launcher with exit code " + + launcher.getServiceExitCode()); + } catch (UnknownHostException expected) { + //expected + } + + } + + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestCommonArgParsing.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestCommonArgParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestCommonArgParsing.java new file mode 100644 index 0000000..ec6dbb8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestCommonArgParsing.java @@ -0,0 +1,522 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.client; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.slider.api.ResourceKeys; +import org.apache.slider.api.RoleKeys; +import org.apache.slider.common.SliderXmlConfKeys; +import org.apache.slider.common.params.AbstractClusterBuildingActionArgs; +import org.apache.slider.common.params.ActionBuildArgs; +import org.apache.slider.common.params.ActionCreateArgs; +import org.apache.slider.common.params.ActionDestroyArgs; +import org.apache.slider.common.params.ActionExistsArgs; +import org.apache.slider.common.params.ActionFlexArgs; +import org.apache.slider.common.params.ActionFreezeArgs; +import org.apache.slider.common.params.ActionListArgs; +import org.apache.slider.common.params.ActionStatusArgs; +import org.apache.slider.common.params.ActionThawArgs; +import org.apache.slider.common.params.ActionUpdateArgs; +import org.apache.slider.common.params.ArgOps; +import org.apache.slider.common.params.Arguments; +import org.apache.slider.common.params.ClientArgs; +import org.apache.slider.common.params.SliderActions; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; +import org.apache.slider.core.exceptions.ErrorStrings; +import org.apache.slider.core.exceptions.SliderException; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** + * Test handling of common arguments, specifically how things get split up. + */ +public class TestCommonArgParsing implements SliderActions, Arguments { + + + public static final String CLUSTERNAME = "clustername"; + + @Test + public void testCreateActionArgs() throws Throwable { + ClientArgs clientArgs = createClientArgs(Arrays.asList(ACTION_CREATE, + "cluster1")); + assertEquals("cluster1", clientArgs.getClusterName()); + } + + @Test + public void testCreateFailsNoClustername() throws Throwable { + assertParseFails(Arrays.asList(ACTION_CREATE)); + } + + @Test + public void testCreateFailsTwoClusternames() throws Throwable { + assertParseFails(Arrays.asList( + ACTION_CREATE, + "c1", + "c2" + )); + } + + @Test + public void testHelp() throws Throwable { + ClientArgs clientArgs = createClientArgs(Arrays.asList(ACTION_HELP)); + assertNull(clientArgs.getClusterName()); + } + + @Test + public void testSliderBasePath() throws Throwable { + ClientArgs clientArgs = createClientArgs(Arrays.asList(ACTION_LIST, + ARG_BASE_PATH, "/projects/slider/clusters")); + assertEquals(new Path("/projects/slider/clusters"), + clientArgs.getBasePath()); + } + + @Test + public void testNoSliderBasePath() throws Throwable { + ClientArgs clientArgs = createClientArgs(Arrays.asList(ACTION_LIST)); + assertNull(clientArgs.getBasePath()); + } + + @Test + public void testListNoClusternames() throws Throwable { + ClientArgs clientArgs = createClientArgs(Arrays.asList(ACTION_LIST)); + assertNull(clientArgs.getClusterName()); + } + + @Test + public void testListNoClusternamesDefinition() throws Throwable { + ClientArgs clientArgs = createClientArgs(Arrays.asList( + ACTION_LIST, + ARG_DEFINE, + "fs.default.FS=file://localhost" + )); + assertNull(clientArgs.getClusterName()); + } + + @Test + public void testList1Clustername() throws Throwable { + ClientArgs ca = createClientArgs(Arrays.asList(ACTION_LIST, "cluster1")); + assertEquals("cluster1", ca.getClusterName()); + assertTrue(ca.getCoreAction() instanceof ActionListArgs); + } + + @Test + public void testListFailsTwoClusternames() throws Throwable { + assertParseFails(Arrays.asList( + ACTION_LIST, + "c1", + "c2" + )); + } + + @Test + public void testDefinitions() throws Throwable { + ClientArgs ca = createClientArgs(Arrays.asList( + ACTION_CREATE, + CLUSTERNAME, + "-D", "yarn.resourcemanager.principal=yarn/server@LOCAL", + "-D", "dfs.datanode.kerberos.principal=hdfs/server@LOCAL" + )); + Configuration conf = new Configuration(false); + ca.applyDefinitions(conf); + assertEquals(CLUSTERNAME, ca.getClusterName()); + assertNull(conf.get(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH)); + SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL); + SliderUtils.verifyPrincipalSet( + conf, + SliderXmlConfKeys.DFS_DATANODE_KERBEROS_PRINCIPAL_KEY); + + } + + @Test + public void testDefinitionsSettingBaseSliderDir() throws Throwable { + ClientArgs ca = createClientArgs(Arrays.asList( + ACTION_CREATE, + CLUSTERNAME, + "--basepath", "/projects/slider/clusters", + "-D", "yarn.resourcemanager.principal=yarn/server@LOCAL", + "-D", "dfs.datanode.kerberos.principal=hdfs/server@LOCAL" + )); + Configuration conf = new Configuration(false); + ca.applyDefinitions(conf); + assertEquals(CLUSTERNAME, ca.getClusterName()); + assertEquals("/projects/slider/clusters", conf.get(SliderXmlConfKeys + .KEY_SLIDER_BASE_PATH)); + SliderUtils.verifyPrincipalSet(conf, YarnConfiguration.RM_PRINCIPAL); + SliderUtils.verifyPrincipalSet(conf, SliderXmlConfKeys + .DFS_DATANODE_KERBEROS_PRINCIPAL_KEY); + + } + + /** + * Test a start command. + * @throws Throwable + */ + @Test + public void testComplexThaw() throws Throwable { + ClientArgs ca = createClientArgs(Arrays.asList( + ACTION_START, + "--manager", "rhel:8032", + "--filesystem", "hdfs://rhel:9090", + "-S", "java.security.krb5.realm=LOCAL", + "-S", "java.security.krb5.kdc=rhel", + "-D", "yarn.resourcemanager.principal=yarn/rhel@LOCAL", + "-D", "namenode.resourcemanager.principal=hdfs/rhel@LOCAL", + "cl1" + )); + assertEquals("cl1", ca.getClusterName()); + assertTrue(ca.getCoreAction() instanceof ActionThawArgs); + } + + /** + * Test a force kill command where the app comes at the end of the line. + * @throws Throwable + * + */ + @Test + public void testStatusSplit() throws Throwable { + + String appId = "application_1381252124398_0013"; + ClientArgs ca = createClientArgs(Arrays.asList( + ACTION_STATUS, + "--manager", "rhel:8032", + "--filesystem", "hdfs://rhel:9090", + "-S", "java.security.krb5.realm=LOCAL", + "-S", "java.security.krb5.kdc=rhel", + "-D", "yarn.resourcemanager.principal=yarn/rhel@LOCAL", + "-D", "namenode.resourcemanager.principal=hdfs/rhel@LOCAL", + appId + )); + assertEquals(appId, ca.getClusterName()); + } + + @Test + public void testFreezeFailsNoArg() throws Throwable { + assertParseFails(Arrays.asList( + ACTION_STOP + )); + } + + @Test + public void testFreezeWorks1Arg() throws Throwable { + ClientArgs ca = createClientArgs(Arrays.asList( + ACTION_STOP, + CLUSTERNAME + )); + assertEquals(CLUSTERNAME, ca.getClusterName()); + assertTrue(ca.getCoreAction() instanceof ActionFreezeArgs); + } + + @Test + public void testFreezeFails2Arg() throws Throwable { + assertParseFails(Arrays.asList( + ACTION_STOP, "cluster", "cluster2" + )); + } + + @Test + public void testFreezeForceWaitAndMessage() throws Throwable { + ClientArgs ca = createClientArgs(Arrays.asList( + ACTION_STOP, CLUSTERNAME, + ARG_FORCE, + ARG_WAIT, "0", + ARG_MESSAGE, "explanation" + )); + assertEquals(CLUSTERNAME, ca.getClusterName()); + assertTrue(ca.getCoreAction() instanceof ActionFreezeArgs); + ActionFreezeArgs freezeArgs = (ActionFreezeArgs) ca.getCoreAction(); + assertEquals("explanation", freezeArgs.message); + assertTrue(freezeArgs.force); + } + + @Test + public void testGetStatusWorks1Arg() throws Throwable { + ClientArgs ca = createClientArgs(Arrays.asList( + ACTION_STATUS, + CLUSTERNAME + )); + assertEquals(CLUSTERNAME, ca.getClusterName()); + assertTrue(ca.getCoreAction() instanceof ActionStatusArgs); + } + + @Test + public void testExistsWorks1Arg() throws Throwable { + ClientArgs ca = createClientArgs(Arrays.asList( + ACTION_EXISTS, + CLUSTERNAME, + ARG_LIVE + )); + assertEquals(CLUSTERNAME, ca.getClusterName()); + assertTrue(ca.getCoreAction() instanceof ActionExistsArgs); + assertTrue(ca.getActionExistsArgs().live); + } + + @Test + public void testDestroy1Arg() throws Throwable { + ClientArgs ca = createClientArgs(Arrays.asList( + ACTION_DESTROY, + CLUSTERNAME + )); + assertEquals(CLUSTERNAME, ca.getClusterName()); + assertTrue(ca.getCoreAction() instanceof ActionDestroyArgs); + } + + /** + * Assert that a pass fails with a BadCommandArgumentsException. + * @param argsList + */ + + private void assertParseFails(List argsList) throws SliderException { + try { + ClientArgs clientArgs = createClientArgs(argsList); + Assert.fail("exected an exception, got " + clientArgs); + } catch (BadCommandArgumentsException ignored) { + //expected + } + } + + /** + * Build and parse client args, after adding the base args list. + * @param argsList + */ + public ClientArgs createClientArgs(List<String> argsList) + throws SliderException { + ClientArgs serviceArgs = new ClientArgs(argsList); + serviceArgs.parse(); + return serviceArgs; + } + + public ActionCreateArgs createAction(List<String> argsList) + throws SliderException { + ClientArgs ca = createClientArgs(argsList); + assertEquals(ACTION_CREATE, ca.getAction()); + ActionCreateArgs args = ca.getActionCreateArgs(); + assertNotNull(args); + return args; + } + + @Test + public void testSingleRoleArg() throws Throwable { + ActionCreateArgs createArgs = createAction(Arrays.asList( + ACTION_CREATE, "cluster1", + ARG_COMPONENT, "master", "5" + )); + List<String> tuples = createArgs.getComponentTuples(); + assertEquals(2, tuples.size()); + Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", tuples); + assertEquals("5", roleMap.get("master")); + } + + @Test + public void testNoRoleArg() throws Throwable { + ActionCreateArgs createArgs = createAction(Arrays.asList( + ACTION_CREATE, "cluster1" + )); + List<String> tuples = createArgs.getComponentTuples(); + Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", tuples); + assertNull(roleMap.get("master")); + } + + + @Test + public void testMultiRoleArgBuild() throws Throwable { + ClientArgs ca = createClientArgs(Arrays.asList( + ACTION_BUILD, "cluster1", + ARG_COMPONENT, "master", "1", + ARG_COMPONENT, "worker", "2" + )); + assertEquals(ACTION_BUILD, ca.getAction()); + assertTrue(ca.getCoreAction() instanceof ActionBuildArgs); + assertTrue(ca.getBuildingActionArgs() instanceof ActionBuildArgs); + AbstractClusterBuildingActionArgs args = ca.getActionBuildArgs(); + List<String> tuples = args.getComponentTuples(); + assertEquals(4, tuples.size()); + Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", tuples); + assertEquals("1", roleMap.get("master")); + assertEquals("2", roleMap.get("worker")); + } + + @Test + public void testArgUpdate() throws Throwable { + ClientArgs ca = createClientArgs(Arrays.asList( + ACTION_UPDATE, "cluster1", + ARG_APPDEF, "app.json" + )); + assertEquals(ACTION_UPDATE, ca.getAction()); + assertTrue(ca.getCoreAction() instanceof ActionUpdateArgs); + assertTrue(ca.getActionUpdateArgs() instanceof ActionUpdateArgs); + AbstractClusterBuildingActionArgs args = ca.getActionUpdateArgs(); + assertNotNull(args.appDef); + } + + @Test + public void testFlexArgs() throws Throwable { + ClientArgs ca = createClientArgs(Arrays.asList( + ACTION_FLEX, "cluster1", + ARG_COMPONENT, "master", "1", + ARG_COMPONENT, "worker", "2" + )); + assertTrue(ca.getCoreAction() instanceof ActionFlexArgs); + List<String> tuples = ca.getActionFlexArgs().getComponentTuples(); + assertEquals(4, tuples.size()); + Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", tuples); + assertEquals("1", roleMap.get("master")); + assertEquals("2", roleMap.get("worker")); + } + + @Test + public void testDuplicateRole() throws Throwable { + ActionCreateArgs createArgs = createAction(Arrays.asList( + ACTION_CREATE, "cluster1", + ARG_COMPONENT, "master", "1", + ARG_COMPONENT, "master", "2" + )); + List<String> tuples = createArgs.getComponentTuples(); + assertEquals(4, tuples.size()); + try { + Map<String, String> roleMap = ArgOps.convertTupleListToMap( + "roles", + tuples); + Assert.fail("got a role map $roleMap not a failure"); + } catch (BadCommandArgumentsException expected) { + assertTrue(expected.getMessage().contains(ErrorStrings + .ERROR_DUPLICATE_ENTRY)); + } + } + + @Test + public void testOddRoleCount() throws Throwable { + ActionCreateArgs createArgs = createAction(Arrays.asList( + ACTION_CREATE, "cluster1", + ARG_COMPONENT, "master", "1", + ARG_COMPONENT, "master", "2" + )); + List<String> tuples = createArgs.getComponentTuples(); + tuples.add("loggers"); + assertEquals(5, tuples.size()); + try { + Map<String, String> roleMap = ArgOps.convertTupleListToMap("roles", + tuples); + Assert.fail("got a role map " + roleMap + " not a failure"); + } catch (BadCommandArgumentsException expected) { + assertTrue(expected.getMessage().contains(ErrorStrings + .ERROR_PARSE_FAILURE)); + } + } + + /** + * Create some role-opt client args, so that multiple tests can use it. + * @return the args + */ + public ActionCreateArgs createRoleOptClientArgs() throws SliderException { + ActionCreateArgs createArgs = createAction(Arrays.asList( + ACTION_CREATE, "cluster1", + ARG_COMPONENT, "master", "1", + ARG_COMP_OPT, "master", "cheese", "swiss", + ARG_COMP_OPT, "master", "env.CHEESE", "cheddar", + ARG_COMP_OPT, "master", ResourceKeys.YARN_CORES, "3", + + ARG_COMPONENT, "worker", "2", + ARG_COMP_OPT, "worker", ResourceKeys.YARN_CORES, "2", + ARG_COMP_OPT, "worker", RoleKeys.JVM_HEAP, "65536", + ARG_COMP_OPT, "worker", "env.CHEESE", "stilton" + )); + return createArgs; + } + + @Test + public void testRoleOptionParse() throws Throwable { + ActionCreateArgs createArgs = createRoleOptClientArgs(); + Map<String, Map<String, String>> tripleMaps = createArgs.getCompOptionMap(); + Map<String, String> workerOpts = tripleMaps.get("worker"); + assertEquals(3, workerOpts.size()); + assertEquals("2", workerOpts.get(ResourceKeys.YARN_CORES)); + assertEquals("65536", workerOpts.get(RoleKeys.JVM_HEAP)); + + Map<String, String> masterOpts = tripleMaps.get("master"); + assertEquals(3, masterOpts.size()); + assertEquals("3", masterOpts.get(ResourceKeys.YARN_CORES)); + + } + + @Test + public void testRoleOptionsMerge() throws Throwable { + ActionCreateArgs createArgs = createRoleOptClientArgs(); + + Map<String, Map<String, String>> roleOpts = createArgs.getCompOptionMap(); + + Map<String, Map<String, String>> clusterRoleMap = createEnvMap(); + SliderUtils.applyCommandLineRoleOptsToRoleMap(clusterRoleMap, roleOpts); + + Map<String, String> masterOpts = clusterRoleMap.get("master"); + assertEquals("swiss", masterOpts.get("cheese")); + + Map<String, String> workerOpts = clusterRoleMap.get("worker"); + assertEquals("stilton", workerOpts.get("env.CHEESE")); + } + + @Test + public void testEnvVariableApply() throws Throwable { + ActionCreateArgs createArgs = createRoleOptClientArgs(); + + + Map<String, Map<String, String>> roleOpts = createArgs.getCompOptionMap(); + + Map<String, Map<String, String>> clusterRoleMap = createEnvMap(); + SliderUtils.applyCommandLineRoleOptsToRoleMap(clusterRoleMap, roleOpts); + + Map<String, String> workerOpts = clusterRoleMap.get("worker"); + assertEquals("stilton", workerOpts.get("env.CHEESE")); + + Map<String, String> envmap = SliderUtils.buildEnvMap(workerOpts); + assertEquals("stilton", envmap.get("CHEESE")); + + } + + /** + * Static compiler complaining about matching LinkedHashMap with Map, + * so some explicit creation here. + * @return a map of maps + */ + public Map<String, Map<String, String>> createEnvMap() { + + Map<String, String> cheese = new HashMap<>(); + cheese.put("cheese", "french"); + Map<String, String> envCheese = new HashMap<>(); + envCheese.put("env.CHEESE", "french"); + Map<String, Map<String, String>> envMap = new HashMap<>(); + envMap.put("master", cheese); + envMap.put("worker", envCheese); + return envMap; + } + + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java new file mode 100644 index 0000000..07d8c10 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestKeytabCommandOptions.java @@ -0,0 +1,405 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.client; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.log4j.AppenderSkeleton; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.LoggingEvent; +import org.apache.slider.common.params.Arguments; +import org.apache.slider.common.params.ClientArgs; +import org.apache.slider.common.tools.SliderFileSystem; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.core.exceptions.BadCommandArgumentsException; +import org.apache.slider.core.exceptions.SliderException; +import org.apache.slider.core.main.ServiceLauncher; +import org.apache.slider.utils.SliderTestBase; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +/** + * Test a keytab installation. + */ +public class TestKeytabCommandOptions extends SliderTestBase { + + private static SliderFileSystem testFileSystem; + + @Before + public void setupFilesystem() throws IOException { + org.apache.hadoop.fs.FileSystem fileSystem = new RawLocalFileSystem(); + YarnConfiguration configuration = SliderUtils.createConfiguration(); + fileSystem.setConf(configuration); + testFileSystem = new SliderFileSystem(fileSystem, configuration); + File testFolderDir = new File(testFileSystem + .buildKeytabInstallationDirPath("").toUri().getPath()); + FileUtils.deleteDirectory(testFolderDir); + } + + @Test + public void testInstallKeytab() throws Throwable { + // create a mock keytab file + File localKeytab = + FileUtil.createLocalTempFile(getTempLocation(), "test", true); + String contents = UUID.randomUUID().toString(); + FileUtils.write(localKeytab, contents); + YarnConfiguration conf = SliderUtils.createConfiguration(); + ServiceLauncher launcher = launch(TestSliderClient.class, + conf, + Arrays.asList( + ClientArgs.ACTION_KEYTAB, + ClientArgs.ARG_KEYTABINSTALL, + ClientArgs.ARG_KEYTAB, + localKeytab.getAbsolutePath(), + Arguments.ARG_FOLDER, + "testFolder")); + Path installedPath = new Path(testFileSystem + .buildKeytabInstallationDirPath("testFolder"), localKeytab.getName()); + File installedKeytab = new File(installedPath.toUri().getPath()); + assertTrue(installedKeytab.exists()); + assertEquals(FileUtils.readFileToString(installedKeytab), + FileUtils.readFileToString(localKeytab)); + } + + @Test + public void testInstallThenDeleteKeytab() throws Throwable { + // create a mock keytab file + File localKeytab = + FileUtil.createLocalTempFile(getTempLocation(), "test", true); + String contents = UUID.randomUUID().toString(); + FileUtils.write(localKeytab, contents); + YarnConfiguration conf = SliderUtils.createConfiguration(); + ServiceLauncher launcher = launch(TestSliderClient.class, + conf, + Arrays.asList( + ClientArgs.ACTION_KEYTAB, + ClientArgs.ARG_KEYTABINSTALL, + ClientArgs.ARG_KEYTAB, + localKeytab.getAbsolutePath(), + Arguments.ARG_FOLDER, + "testFolder")); + Path installedPath = new Path(testFileSystem + .buildKeytabInstallationDirPath("testFolder"), localKeytab.getName()); + File installedKeytab = new File(installedPath.toUri().getPath()); + assertTrue(installedKeytab.exists()); + assertEquals(FileUtils.readFileToString(installedKeytab), + FileUtils.readFileToString(localKeytab)); + + launcher = launch(TestSliderClient.class, + conf, + Arrays.asList( + ClientArgs.ACTION_KEYTAB, + ClientArgs.ARG_KEYTABDELETE, + ClientArgs.ARG_KEYTAB, + localKeytab.getName(), + Arguments.ARG_FOLDER, + "testFolder")); + + assertFalse(installedKeytab.exists()); + + } + + @Test + public void testInstallThenListKeytab() throws Throwable { + // create a mock keytab file + File localKeytab = + FileUtil.createLocalTempFile(getTempLocation(), "test", true); + String contents = UUID.randomUUID().toString(); + FileUtils.write(localKeytab, contents); + YarnConfiguration conf = SliderUtils.createConfiguration(); + ServiceLauncher launcher = launch(TestSliderClient.class, + conf, + Arrays.asList( + ClientArgs.ACTION_KEYTAB, + ClientArgs.ARG_KEYTABINSTALL, + ClientArgs.ARG_KEYTAB, + localKeytab.getAbsolutePath(), + Arguments.ARG_FOLDER, + "testFolder")); + Path installedPath = new Path(testFileSystem + .buildKeytabInstallationDirPath("testFolder"), localKeytab.getName()); + File installedKeytab = new File(installedPath.toUri().getPath()); + assertTrue(installedKeytab.exists()); + assertEquals(FileUtils.readFileToString(installedKeytab), + FileUtils.readFileToString(localKeytab)); + + // install an additional copy into another folder to test listing + launcher = launch(TestSliderClient.class, + conf, + Arrays.asList( + ClientArgs.ACTION_KEYTAB, + ClientArgs.ARG_KEYTABINSTALL, + ClientArgs.ARG_KEYTAB, + localKeytab.getAbsolutePath(), + Arguments.ARG_FOLDER, + "testFolder2")); + + TestAppender testAppender = new TestAppender(); + + Logger.getLogger(SliderClient.class).addAppender(testAppender); + + try { + launcher = launch(TestSliderClient.class, + conf, + Arrays.asList( + ClientArgs.ACTION_KEYTAB, + ClientArgs.ARG_KEYTABLIST) + ); + assertEquals(3, testAppender.events.size()); + String msg = (String) testAppender.events.get(1).getMessage(); + assertTrue(msg.contains("/.slider/keytabs/testFolder")); + assertTrue(msg.endsWith(installedKeytab.getName())); + msg = (String) testAppender.events.get(2).getMessage(); + assertTrue(msg.contains("/.slider/keytabs/testFolder")); + assertTrue(msg.endsWith(installedKeytab.getName())); + } finally { + Logger.getLogger(SliderClient.class).removeAppender(testAppender); + } + + // now listing while specifying the folder name + testAppender = new TestAppender(); + + Logger.getLogger(SliderClient.class).addAppender(testAppender); + + try { + launcher = launch(TestSliderClient.class, + conf, + Arrays.asList( + ClientArgs.ACTION_KEYTAB, + ClientArgs.ARG_KEYTABLIST, + Arguments.ARG_FOLDER, + "testFolder")); + assertEquals(2, testAppender.events.size()); + String msg = (String) testAppender.events.get(1).getMessage(); + assertTrue(msg.contains("/.slider/keytabs/testFolder/" + + installedKeytab.getName())); + } finally { + Logger.getLogger(SliderClient.class).removeAppender(testAppender); + } + } + + @Test + public void testDeleteNonExistentKeytab() throws Throwable { + // create a mock keytab file + YarnConfiguration conf = SliderUtils.createConfiguration(); + try { + ServiceLauncher launcher = launch(TestSliderClient.class, + conf, + Arrays.asList( + ClientArgs.ACTION_KEYTAB, + ClientArgs.ARG_KEYTABDELETE, + ClientArgs.ARG_KEYTAB, + "HeyIDontExist.keytab", + Arguments.ARG_FOLDER, + "testFolder")); + fail("expected BadCommandArgumentsException from launch"); + } catch (BadCommandArgumentsException e) { + // expected + } + } + + @Test + public void testInstallKeytabWithNoFolder() throws Throwable { + // create a mock keytab file + File localKeytab = + FileUtil.createLocalTempFile(getTempLocation(), "test", true); + String contents = UUID.randomUUID().toString(); + FileUtils.write(localKeytab, contents); + YarnConfiguration conf = SliderUtils.createConfiguration(); + try { + ServiceLauncher launcher = launch(TestSliderClient.class, + conf, + Arrays.asList( + ClientArgs.ACTION_KEYTAB, + ClientArgs.ARG_KEYTABINSTALL, + ClientArgs.ARG_KEYTAB, + localKeytab.getAbsolutePath())); + fail("expected BadCommandArgumentsException from launch"); + } catch (BadCommandArgumentsException e) { + // expected + } + } + + @Test + public void testInstallKeytabWithNoKeytab() throws Throwable { + // create a mock keytab file + File localKeytab = + FileUtil.createLocalTempFile(getTempLocation(), "test", true); + String contents = UUID.randomUUID().toString(); + FileUtils.write(localKeytab, contents); + YarnConfiguration conf = SliderUtils.createConfiguration(); + try { + ServiceLauncher launcher = launch(TestSliderClient.class, + conf, + Arrays.asList( + ClientArgs.ACTION_KEYTAB, + ClientArgs.ARG_KEYTABINSTALL, + ClientArgs.ARG_FOLDER, + "testFolder")); + fail("expected BadCommandArgumentsException from launch"); + } catch (BadCommandArgumentsException e) { + // expected + } + } + + @Test + public void testInstallKeytabAllowingOverwrite() throws Throwable { + // create a mock keytab file + File localKeytab = + FileUtil.createLocalTempFile(getTempLocation(), "test", true); + String contents = UUID.randomUUID().toString(); + FileUtils.write(localKeytab, contents); + YarnConfiguration conf = SliderUtils.createConfiguration(); + ServiceLauncher launcher = launch(TestSliderClient.class, + conf, + Arrays.asList( + ClientArgs.ACTION_KEYTAB, + ClientArgs.ARG_KEYTABINSTALL, + ClientArgs.ARG_KEYTAB, + localKeytab.getAbsolutePath(), + Arguments.ARG_FOLDER, + "testFolder")); + Path installedPath = new Path(testFileSystem + .buildKeytabInstallationDirPath("testFolder"), localKeytab.getName()); + File installedKeytab = new File(installedPath.toUri().getPath()); + assertTrue(installedKeytab.exists()); + assertEquals(FileUtils.readFileToString(installedKeytab), FileUtils + .readFileToString(localKeytab)); + launcher = launch(TestSliderClient.class, + conf, + Arrays.asList( + ClientArgs.ACTION_KEYTAB, + ClientArgs.ARG_KEYTABINSTALL, + ClientArgs.ARG_KEYTAB, + localKeytab.getAbsolutePath(), + Arguments.ARG_FOLDER, + "testFolder", + Arguments.ARG_OVERWRITE) + ); + assertTrue(installedKeytab.exists()); + assertEquals(FileUtils.readFileToString(installedKeytab), + FileUtils.readFileToString(localKeytab)); + } + + @Test + public void testInstallKeytabNotAllowingOverwrite() throws Throwable { + // create a mock keytab file + File localKeytab = + FileUtil.createLocalTempFile(getTempLocation(), "test", true); + String contents = UUID.randomUUID().toString(); + FileUtils.write(localKeytab, contents); + YarnConfiguration conf = SliderUtils.createConfiguration(); + ServiceLauncher launcher = launch(TestSliderClient.class, + conf, + Arrays.asList( + ClientArgs.ACTION_KEYTAB, + ClientArgs.ARG_KEYTABINSTALL, + ClientArgs.ARG_KEYTAB, + localKeytab.getAbsolutePath(), + Arguments.ARG_FOLDER, + "testFolder")); + Path installedPath = new Path(testFileSystem + .buildKeytabInstallationDirPath("testFolder"), localKeytab.getName()); + File installedKeytab = new File(installedPath.toUri().getPath()); + assertTrue(installedKeytab.exists()); + assertEquals(FileUtils.readFileToString(installedKeytab), + FileUtils.readFileToString(localKeytab)); + try { + launcher = launch(TestSliderClient.class, + conf, + Arrays.asList( + ClientArgs.ACTION_KEYTAB, + ClientArgs.ARG_KEYTABINSTALL, + ClientArgs.ARG_KEYTAB, + localKeytab.getAbsolutePath(), + Arguments.ARG_FOLDER, + "testFolder")); + fail("expected BadCommandArgumentsException from launch"); + } catch (BadCommandArgumentsException e) { + // expected + } + } + + @Test + public void testInstallKeytabWithMissingKeytab() throws Throwable { + // create a mock keytab file + YarnConfiguration conf = SliderUtils.createConfiguration(); + try { + ServiceLauncher launcher = launch(TestSliderClient.class, + conf, + Arrays.asList( + ClientArgs.ACTION_KEYTAB, + ClientArgs.ARG_KEYTABINSTALL, + ClientArgs.ARG_KEYTAB, + "HeyIDontExist.keytab", + Arguments.ARG_FOLDER, + "testFolder")); + fail("expected BadCommandArgumentsException from launch"); + } catch (BadCommandArgumentsException e) { + // expected + } + } + + private File getTempLocation() { + return new File(System.getProperty("user.dir") + "/target"); + } + + /** + * Test SliderClient with overridden filesystem. + */ + public static class TestSliderClient extends SliderClient { + public TestSliderClient() { + super(); + } + + @Override + protected void initHadoopBinding() throws IOException, SliderException { + sliderFileSystem = testFileSystem; + } + + } + + /** + * Appender that captures logging events. + */ + public static class TestAppender extends AppenderSkeleton { + private List<LoggingEvent> events = new ArrayList<>(); + + public void close() {} + + public boolean requiresLayout() { + return false; + } + + @Override + protected void append(LoggingEvent event) { + events.add(event); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderClientMethods.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderClientMethods.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderClientMethods.java new file mode 100644 index 0000000..32208ab --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderClientMethods.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.client; + +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.slider.common.SliderXmlConfKeys; +import org.apache.slider.common.tools.SliderUtils; +import org.apache.slider.server.appmaster.model.mock.MockApplicationId; +import org.apache.slider.utils.SliderTestBase; +import org.easymock.EasyMock; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.powermock.api.easymock.PowerMock; +import org.powermock.core.classloader.annotations.PrepareForTest; +import org.powermock.modules.junit4.PowerMockRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; + +/** + * Test slider client methods. + */ +@RunWith(PowerMockRunner.class) +@PrepareForTest(SliderUtils.class) +public class TestSliderClientMethods extends SliderTestBase { + protected static final Logger LOG = + LoggerFactory.getLogger(TestSliderClientMethods.class); + + static final String AM_ENV = "LD_LIBRARY_PATH"; + static final String PLACEHOLDER_KEY = "${distro.version}"; + static final String PLACEHOLDER_SYSTEM_KEY = "DISTRO_VERSION"; + static final String PLACEHOLDER_VALUE = "1.0.0"; + static final String AM_ENV_2 = "PATH"; + static final String PLACEHOLDER_KEY_2 = "${native.version}"; + static final String PLACEHOLDER_SYSTEM_KEY_2 = "NATIVE_VERSION"; + static final String PLACEHOLDER_VALUE_2 = "2.0.0"; + + @Test + public void testGeneratePlaceholderKeyValueMap() throws Throwable { + TestSliderClient testSliderClient = new TestSliderClient(); + + PowerMock.mockStatic(System.class); + EasyMock.expect(SliderUtils.getSystemEnv(PLACEHOLDER_SYSTEM_KEY)) + .andReturn(PLACEHOLDER_VALUE).anyTimes(); + PowerMock.replayAll(); + + Map<String, String> placeholders = testSliderClient + .generatePlaceholderKeyValueMap(AM_ENV + "=/usr/lib/" + + PLACEHOLDER_KEY); + Assert.assertTrue(placeholders.containsKey(PLACEHOLDER_KEY)); + Assert.assertEquals("Should be equal", PLACEHOLDER_VALUE, + placeholders.get(PLACEHOLDER_KEY)); + + PowerMock.verifyAll(); + LOG.info("Placeholders = {}", placeholders); + } + + @Test + public void testSetAmLaunchEnv() throws Throwable { + TestSliderClient testSliderClient = new TestSliderClient(); + YarnConfiguration conf = SliderUtils.createConfiguration(); + conf.set(SliderXmlConfKeys.KEY_AM_LAUNCH_ENV, AM_ENV + "=/usr/lib/" + + PLACEHOLDER_KEY); + + PowerMock.mockStatic(System.class); + EasyMock.expect(SliderUtils.getSystemEnv(PLACEHOLDER_SYSTEM_KEY)) + .andReturn(PLACEHOLDER_VALUE); + PowerMock.replayAll(); + + Map<String, String> amLaunchEnv = testSliderClient.getAmLaunchEnv(conf); + Assert.assertNotNull(amLaunchEnv); + Assert.assertNotNull(amLaunchEnv.get(AM_ENV)); + Assert.assertEquals("Should be equal", amLaunchEnv.get(AM_ENV), + (Shell.WINDOWS ? "%" + AM_ENV + "%;" : "$" + AM_ENV + ":") + + "/usr/lib/" + PLACEHOLDER_VALUE); + + PowerMock.verifyAll(); + LOG.info("amLaunchEnv = {}", amLaunchEnv); + } + + @Test + public void testSetAmLaunchEnvMulti() throws Throwable { + TestSliderClient testSliderClient = new TestSliderClient(); + YarnConfiguration conf = SliderUtils.createConfiguration(); + conf.set(SliderXmlConfKeys.KEY_AM_LAUNCH_ENV, AM_ENV + "=/usr/lib/" + + PLACEHOLDER_KEY + "," + AM_ENV_2 + "=/usr/bin/" + PLACEHOLDER_KEY_2); + + PowerMock.mockStatic(System.class); + EasyMock.expect(SliderUtils.getSystemEnv(PLACEHOLDER_SYSTEM_KEY)) + .andReturn(PLACEHOLDER_VALUE); + EasyMock.expect(SliderUtils.getSystemEnv(PLACEHOLDER_SYSTEM_KEY_2)) + .andReturn(PLACEHOLDER_VALUE_2); + PowerMock.replayAll(); + + Map<String, String> amLaunchEnv = testSliderClient.getAmLaunchEnv(conf); + Assert.assertNotNull(amLaunchEnv); + Assert.assertEquals("Should have 2 envs", amLaunchEnv.size(), 2); + Assert.assertNotNull(amLaunchEnv.get(AM_ENV)); + Assert.assertEquals("Should be equal", amLaunchEnv.get(AM_ENV), + (Shell.WINDOWS ? "%" + AM_ENV + "%;" : "$" + AM_ENV + ":") + + "/usr/lib/" + PLACEHOLDER_VALUE); + Assert.assertNotNull(amLaunchEnv.get(AM_ENV_2)); + Assert.assertEquals("Should be equal", amLaunchEnv.get(AM_ENV_2), + (Shell.WINDOWS ? "%" + AM_ENV_2 + "%;" : "$" + AM_ENV_2 + ":") + + "/usr/bin/" + PLACEHOLDER_VALUE_2); + + PowerMock.verifyAll(); + LOG.info("amLaunchEnv = " + amLaunchEnv); + } + + static class TestSliderClient extends SliderClient { + @Override + public ApplicationId submitApplication(ApplicationSubmissionContext + context) + throws YarnException, IOException { + return new MockApplicationId(1); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderTokensCommand.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderTokensCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderTokensCommand.java new file mode 100644 index 0000000..f649ab7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/client/TestSliderTokensCommand.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.client; + +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.slider.common.params.ActionTokensArgs; +import org.apache.slider.common.params.Arguments; +import org.apache.slider.common.params.SliderActions; +import org.apache.slider.core.exceptions.BadClusterStateException; +import org.apache.slider.core.exceptions.NotFoundException; +import org.apache.slider.utils.SliderTestBase; +import org.junit.Test; + +import java.util.Arrays; + +/** + * Test the argument parsing/validation logic. + */ +public class TestSliderTokensCommand extends SliderTestBase { + + private static YarnConfiguration config = createTestConfig(); + + public static YarnConfiguration createTestConfig() { + YarnConfiguration configuration = new YarnConfiguration(); + configuration.set(YarnConfiguration.RM_ADDRESS, "127.0.0.1:8032"); + return configuration; + } + + @Test + public void testBadSourceArgs() throws Throwable { + launchExpectingException(SliderClient.class, + config, + ActionTokensArgs.DUPLICATE_ARGS, + Arrays.asList(SliderActions.ACTION_TOKENS, + Arguments.ARG_SOURCE, "target/tokens.bin", + Arguments.ARG_OUTPUT, "target/tokens.bin" + )); + } + + @Test + public void testKTNoPrincipal() throws Throwable { + launchExpectingException(SliderClient.class, + config, + ActionTokensArgs.MISSING_KT_PROVIDER, + Arrays.asList(SliderActions.ACTION_TOKENS, + Arguments.ARG_KEYTAB, "target/keytab" + )); + } + + @Test + public void testPrincipalNoKT() throws Throwable { + launchExpectingException(SliderClient.class, + config, + ActionTokensArgs.MISSING_KT_PROVIDER, + Arrays.asList(SliderActions.ACTION_TOKENS, + Arguments.ARG_PRINCIPAL, "bob@REALM" + )); + } + + /** + * A missing keytab is an error. + * @throws Throwable + */ + @Test + public void testMissingKT() throws Throwable { + Throwable ex = launchExpectingException(SliderClient.class, + config, + TokensOperation.E_NO_KEYTAB, + Arrays.asList(SliderActions.ACTION_TOKENS, + Arguments.ARG_PRINCIPAL, "bob@REALM", + Arguments.ARG_KEYTAB, "target/keytab" + )); + if (!(ex instanceof NotFoundException)) { + throw ex; + } + } + + @Test + public void testMissingSourceFile() throws Throwable { + Throwable ex = launchExpectingException(SliderClient.class, + config, + TokensOperation.E_MISSING_SOURCE_FILE, + Arrays.asList(SliderActions.ACTION_TOKENS, + Arguments.ARG_SOURCE, "target/tokens.bin" + )); + if (!(ex instanceof NotFoundException)) { + throw ex; + } + } + + @Test + public void testListHarmlessWhenInsecure() throws Throwable { + execSliderCommand(0, config, Arrays.asList(SliderActions.ACTION_TOKENS)); + } + + @Test + public void testCreateFailsWhenInsecure() throws Throwable { + Throwable ex = launchExpectingException(SliderClient.class, + config, + TokensOperation.E_INSECURE, + Arrays.asList(SliderActions.ACTION_TOKENS, + Arguments.ARG_OUTPUT, "target/tokens.bin" + )); + if (!(ex instanceof BadClusterStateException)) { + throw ex; + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestClusterNames.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestClusterNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestClusterNames.java new file mode 100644 index 0000000..efd0c2f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestClusterNames.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.tools; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; +import java.util.List; + +/** + * Test cluster name validation. + */ +public class TestClusterNames { + + void assertValidName(String name) { + boolean valid = SliderUtils.isClusternameValid(name); + Assert.assertTrue("Clustername '" + name + "' mistakenly declared invalid", + valid); + } + + void assertInvalidName(String name) { + boolean valid = SliderUtils.isClusternameValid(name); + Assert.assertFalse("Clustername '\" + name + \"' mistakenly declared valid", + valid); + } + + void assertInvalid(List<String> names) { + for (String name : names) { + assertInvalidName(name); + } + } + + void assertValid(List<String> names) { + for (String name : names) { + assertValidName(name); + } + } + + @Test + public void testEmptyName() throws Throwable { + assertInvalidName(""); + } + + @Test + public void testSpaceName() throws Throwable { + assertInvalidName(" "); + } + + + @Test + public void testLeadingHyphen() throws Throwable { + assertInvalidName("-hyphen"); + } + + @Test + public void testTitleLetters() throws Throwable { + assertInvalidName("Title"); + } + + @Test + public void testCapitalLetters() throws Throwable { + assertInvalidName("UPPER-CASE-CLUSTER"); + } + + @Test + public void testInnerBraced() throws Throwable { + assertInvalidName("a[a"); + } + + @Test + public void testLeadingBrace() throws Throwable { + assertInvalidName("["); + } + + @Test + public void testNonalphaLeadingChars() throws Throwable { + assertInvalid(Arrays.asList( + "[a", "#", "@", "=", "*", "." + )); + } + + @Test + public void testNonalphaInnerChars() throws Throwable { + assertInvalid(Arrays.asList( + "a[a", "b#", "c@", "d=", "e*", "f.", "g ", "h i" + )); + } + + @Test + public void testClusterValid() throws Throwable { + assertValidName("cluster"); + } + + @Test + public void testValidNames() throws Throwable { + assertValid(Arrays.asList( + "cluster", + "cluster1", + "very-very-very-long-cluster-name", + "c1234567890" + )); + + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelper.java new file mode 100644 index 0000000..45c6118 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelper.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.tools; + +import org.apache.hadoop.conf.Configuration; +import org.apache.slider.utils.YarnMiniClusterTestBase; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.util.Map; + +/** + * Test config helper. + */ +public class TestConfigHelper extends YarnMiniClusterTestBase { + + @Test + public void testConfigLoaderIteration() throws Throwable { + + String xml = "<?xml version=\"1.0\" encoding=\"UTF-8\" " + + "standalone=\"no\"?><configuration><property><name>key</name>" + + "<value>value</value><source>programatically</source></property>" + + "</configuration>"; + InputStream ins = new ByteArrayInputStream(xml.getBytes("UTF8")); + Configuration conf = new Configuration(false); + conf.addResource(ins); + Configuration conf2 = new Configuration(false); + for (Map.Entry<String, String> entry : conf) { + conf2.set(entry.getKey(), entry.getValue(), "src"); + } + + } + + @Test + public void testConfigDeprecation() throws Throwable { + ConfigHelper.registerDeprecatedConfigItems(); + Configuration conf = new Configuration(false); + // test deprecated items here + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelperHDFS.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelperHDFS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelperHDFS.java new file mode 100644 index 0000000..f9a58d4 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestConfigHelperHDFS.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.tools; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.slider.utils.YarnMiniClusterTestBase; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.net.URI; + +/** + * Test config helper loading configuration from HDFS. + */ +public class TestConfigHelperHDFS extends YarnMiniClusterTestBase { + private static final Logger LOG = + LoggerFactory.getLogger(TestConfigHelperHDFS.class); + + @Test + public void testConfigHelperHDFS() throws Throwable { + YarnConfiguration config = getConfiguration(); + createMiniHDFSCluster("testConfigHelperHDFS", config); + + Configuration conf = new Configuration(false); + conf.set("key", "value"); + URI fsURI = new URI(getFsDefaultName()); + Path root = new Path(fsURI); + Path confPath = new Path(root, "conf.xml"); + FileSystem dfs = FileSystem.get(fsURI, config); + ConfigHelper.saveConfig(dfs, confPath, conf); + //load time + Configuration loaded = ConfigHelper.loadConfiguration(dfs, confPath); + LOG.info(ConfigHelper.dumpConfigToString(loaded)); + assertEquals("value", loaded.get("key")); + } + +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
