http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestMarshalling.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestMarshalling.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestMarshalling.java new file mode 100644 index 0000000..14e3b1f --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestMarshalling.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.registry.client.binding; + +import org.apache.hadoop.registry.RegistryTestHelper; +import org.apache.hadoop.registry.client.exceptions.NoRecordException; +import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.hadoop.registry.client.types.ServiceRecordHeader; +import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.rules.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.EOFException; + +/** + * Test record marshalling + */ +public class TestMarshalling extends RegistryTestHelper { + private static final Logger + LOG = LoggerFactory.getLogger(TestMarshalling.class); + + @Rule + public final Timeout testTimeout = new Timeout(10000); + @Rule + public TestName methodName = new TestName(); + private static RegistryUtils.ServiceRecordMarshal marshal; + + @BeforeClass + public static void setupClass() { + marshal = new RegistryUtils.ServiceRecordMarshal(); + } + + @Test + public void testRoundTrip() throws Throwable { + String persistence = PersistencePolicies.PERMANENT; + ServiceRecord record = createRecord(persistence); + record.set("customkey","customvalue"); + record.set("customkey2","customvalue2"); + LOG.info(marshal.toJson(record)); + byte[] bytes = marshal.toBytes(record); + ServiceRecord r2 = marshal.fromBytes("", bytes, 0); + assertMatches(record, r2); + } + + @Test + public void testRoundTripHeaders() throws Throwable { + ServiceRecord record = createRecord(PersistencePolicies.CONTAINER); + byte[] bytes = marshal.toByteswithHeader(record); + ServiceRecord r2 = marshal.fromBytesWithHeader("", bytes); + assertMatches(record, r2); + + } + + @Test(expected = NoRecordException.class) + public void testRoundTripBadHeaders() throws Throwable { + ServiceRecord record = createRecord(PersistencePolicies.APPLICATION); + byte[] bytes = marshal.toByteswithHeader(record); + bytes[1] = 0x01; + marshal.fromBytesWithHeader("src", bytes); + } + + @Test(expected = NoRecordException.class) + public void testUnmarshallHeaderTooShort() throws Throwable { + marshal.fromBytesWithHeader("src", new byte[]{'a'}); + } + + @Test(expected = EOFException.class) + public void testUnmarshallNoBody() throws Throwable { + byte[] bytes = ServiceRecordHeader.getData(); + marshal.fromBytesWithHeader("src", bytes); + } + + + @Test + public void testUnknownFieldsRoundTrip() throws Throwable { + ServiceRecord record = + createRecord(PersistencePolicies.APPLICATION_ATTEMPT); + record.set("key", "value"); + record.set("intval", "2"); + assertEquals("value", record.get("key")); + assertEquals("2", record.get("intval")); + assertNull(record.get("null")); + assertEquals("defval", record.get("null", "defval")); + byte[] bytes = marshal.toByteswithHeader(record); + ServiceRecord r2 = marshal.fromBytesWithHeader("", bytes); + assertEquals("value", r2.get("key")); + assertEquals("2", r2.get("intval")); + } + + @Test + public void testFieldPropagationInCopy() throws Throwable { + ServiceRecord record = + createRecord(PersistencePolicies.APPLICATION_ATTEMPT); + record.set("key", "value"); + record.set("intval", "2"); + ServiceRecord that = new ServiceRecord(record); + assertMatches(record, that); + } + +}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryOperationUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryOperationUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryOperationUtils.java new file mode 100644 index 0000000..b86e3fe --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryOperationUtils.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.registry.client.binding; + +import org.apache.hadoop.security.UserGroupInformation; +import org.junit.Assert; +import org.junit.Test; + +/** + * Tests for the {@link RegistryUtils} class + */ +public class TestRegistryOperationUtils extends Assert { + + @Test + public void testUsernameExtractionEnvVarOverrride() throws Throwable { + String whoami = RegistryUtils.getCurrentUsernameUnencoded("drwho"); + assertEquals("drwho", whoami); + + } + + @Test + public void testUsernameExtractionCurrentuser() throws Throwable { + String whoami = RegistryUtils.getCurrentUsernameUnencoded(""); + String ugiUser = UserGroupInformation.getCurrentUser().getShortUserName(); + + assertEquals(ugiUser, whoami); + + } + + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java new file mode 100644 index 0000000..9a24f1c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.registry.client.binding; + +import static org.apache.hadoop.registry.client.binding.RegistryPathUtils.*; +import java.io.IOException; +import java.util.List; + +import org.apache.hadoop.fs.PathNotFoundException; +import org.apache.hadoop.registry.client.exceptions.InvalidPathnameException; +import org.junit.Assert; +import org.junit.Test; + +public class TestRegistryPathUtils extends Assert { + + + public static final String EURO = "\u20AC"; + + @Test + public void testFormatAscii() throws Throwable { + + String in = "hostname01101101-1"; + assertConverted(in, in); + } + + /* + * Euro symbol + */ + @Test + public void testFormatEuroSymbol() throws Throwable { + assertConverted("xn--lzg", EURO); + } + + @Test + public void testFormatIdempotent() throws Throwable { + assertConverted("xn--lzg", RegistryPathUtils.encodeForRegistry(EURO)); + } + + @Test + public void testFormatCyrillicSpaced() throws Throwable { + assertConverted("xn--pa 3-k4di", "\u0413PA\u0414 3"); + } + + protected void assertConverted(String expected, String in) { + String out = RegistryPathUtils.encodeForRegistry(in); + assertEquals("Conversion of " + in, expected, out); + } + + @Test + public void testPaths() throws Throwable { + assertCreatedPathEquals("/", "/", ""); + assertCreatedPathEquals("/", "", ""); + assertCreatedPathEquals("/", "", "/"); + assertCreatedPathEquals("/", "/", "/"); + + assertCreatedPathEquals("/a", "/a", ""); + assertCreatedPathEquals("/a", "/", "a"); + assertCreatedPathEquals("/a/b", "/a", "b"); + assertCreatedPathEquals("/a/b", "/a/", "b"); + assertCreatedPathEquals("/a/b", "/a", "/b"); + assertCreatedPathEquals("/a/b", "/a", "/b/"); + assertCreatedPathEquals("/a", "/a", "/"); + assertCreatedPathEquals("/alice", "/", "/alice"); + assertCreatedPathEquals("/alice", "/alice", "/"); + } + + + + + @Test + public void testComplexPaths() throws Throwable { + assertCreatedPathEquals("/", "", ""); + assertCreatedPathEquals("/yarn/registry/users/hadoop/org-apache-hadoop", + "/yarn/registry", + "users/hadoop/org-apache-hadoop/"); + } + + + private static void assertCreatedPathEquals(String expected, String base, + String path) throws IOException { + String fullPath = createFullPath(base, path); + assertEquals("\"" + base + "\" + \"" + path + "\" =\"" + fullPath + "\"", + expected, fullPath); + } + + @Test + public void testSplittingEmpty() throws Throwable { + assertEquals(0, split("").size()); + assertEquals(0, split("/").size()); + assertEquals(0, split("///").size()); + } + + + @Test + public void testSplitting() throws Throwable { + assertEquals(1, split("/a").size()); + assertEquals(0, split("/").size()); + assertEquals(3, split("/a/b/c").size()); + assertEquals(3, split("/a/b/c/").size()); + assertEquals(3, split("a/b/c").size()); + assertEquals(3, split("/a/b//c").size()); + assertEquals(3, split("//a/b/c/").size()); + List<String> split = split("//a/b/c/"); + assertEquals("a", split.get(0)); + assertEquals("b", split.get(1)); + assertEquals("c", split.get(2)); + } + + @Test + public void testParentOf() throws Throwable { + assertEquals("/", parentOf("/a")); + assertEquals("/", parentOf("/a/")); + assertEquals("/a", parentOf("/a/b")); + assertEquals("/a/b", parentOf("/a/b/c")); + } + + @Test + public void testLastPathEntry() throws Throwable { + assertEquals("",lastPathEntry("/")); + assertEquals("",lastPathEntry("//")); + assertEquals("c",lastPathEntry("/a/b/c")); + assertEquals("c",lastPathEntry("/a/b/c/")); + } + + @Test(expected = PathNotFoundException.class) + public void testParentOfRoot() throws Throwable { + parentOf("/"); + } + + @Test + public void testValidPaths() throws Throwable { + assertValidPath("/"); + assertValidPath("/a/b/c"); + assertValidPath("/users/drwho/org-apache-hadoop/registry/appid-55-55"); + assertValidPath("/a50"); + } + + @Test + public void testInvalidPaths() throws Throwable { + assertInvalidPath("/a_b"); + assertInvalidPath("/UpperAndLowerCase"); + assertInvalidPath("/space in string"); +// Is this valid? assertInvalidPath("/50"); + } + + + private void assertValidPath(String path) throws InvalidPathnameException { + validateZKPath(path); + } + + + private void assertInvalidPath(String path) throws InvalidPathnameException { + try { + validateElementsAsDNS(path); + fail("path considered valid: " + path); + } catch (InvalidPathnameException expected) { + // expected + } + } + + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/CuratorEventCatcher.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/CuratorEventCatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/CuratorEventCatcher.java new file mode 100644 index 0000000..254ab79 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/CuratorEventCatcher.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.registry.client.impl; + +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.api.BackgroundCallback; +import org.apache.curator.framework.api.CuratorEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * This is a little event catcher for curator asynchronous + * operations. + */ +public class CuratorEventCatcher implements BackgroundCallback { + + private static final Logger LOG = + LoggerFactory.getLogger(CuratorEventCatcher.class); + + public final BlockingQueue<CuratorEvent> + events = new LinkedBlockingQueue<CuratorEvent>(1); + + private final AtomicInteger eventCounter = new AtomicInteger(0); + + + @Override + public void processResult(CuratorFramework client, + CuratorEvent event) throws + Exception { + LOG.info("received {}", event); + eventCounter.incrementAndGet(); + events.put(event); + } + + + public int getCount() { + return eventCounter.get(); + } + + /** + * Blocking operation to take the first event off the queue + * @return the first event on the queue, when it arrives + * @throws InterruptedException if interrupted + */ + public CuratorEvent take() throws InterruptedException { + return events.take(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/TestCuratorService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/TestCuratorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/TestCuratorService.java new file mode 100644 index 0000000..3c8b1d1 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/TestCuratorService.java @@ -0,0 +1,249 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.registry.client.impl; + +import org.apache.curator.framework.api.CuratorEvent; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; +import org.apache.hadoop.fs.PathNotFoundException; +import org.apache.hadoop.service.ServiceOperations; +import org.apache.hadoop.registry.AbstractZKRegistryTest; +import org.apache.hadoop.registry.client.impl.zk.CuratorService; +import org.apache.hadoop.registry.client.impl.zk.RegistrySecurity; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.data.ACL; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; + +/** + * Test the curator service + */ +public class TestCuratorService extends AbstractZKRegistryTest { + private static final Logger LOG = + LoggerFactory.getLogger(TestCuratorService.class); + + + protected CuratorService curatorService; + + public static final String MISSING = "/missing"; + private List<ACL> rootACL; + + @Before + public void startCurator() throws IOException { + createCuratorService(); + } + + @After + public void stopCurator() { + ServiceOperations.stop(curatorService); + } + + /** + * Create an instance + */ + protected void createCuratorService() throws IOException { + curatorService = new CuratorService("curatorService"); + curatorService.init(createRegistryConfiguration()); + curatorService.start(); + rootACL = RegistrySecurity.WorldReadWriteACL; + curatorService.maybeCreate("", CreateMode.PERSISTENT, rootACL, true); + } + + @Test + public void testLs() throws Throwable { + curatorService.zkList("/"); + } + + @Test(expected = PathNotFoundException.class) + public void testLsNotFound() throws Throwable { + List<String> ls = curatorService.zkList(MISSING); + } + + @Test + public void testExists() throws Throwable { + assertTrue(curatorService.zkPathExists("/")); + } + + @Test + public void testExistsMissing() throws Throwable { + assertFalse(curatorService.zkPathExists(MISSING)); + } + + @Test + public void testVerifyExists() throws Throwable { + pathMustExist("/"); + } + + @Test(expected = PathNotFoundException.class) + public void testVerifyExistsMissing() throws Throwable { + pathMustExist("/file-not-found"); + } + + @Test + public void testMkdirs() throws Throwable { + mkPath("/p1", CreateMode.PERSISTENT); + pathMustExist("/p1"); + mkPath("/p1/p2", CreateMode.EPHEMERAL); + pathMustExist("/p1/p2"); + } + + private void mkPath(String path, CreateMode mode) throws IOException { + curatorService.zkMkPath(path, mode, false, + RegistrySecurity.WorldReadWriteACL); + } + + public void pathMustExist(String path) throws IOException { + curatorService.zkPathMustExist(path); + } + + @Test(expected = PathNotFoundException.class) + public void testMkdirChild() throws Throwable { + mkPath("/testMkdirChild/child", CreateMode.PERSISTENT); + } + + @Test + public void testMaybeCreate() throws Throwable { + assertTrue(curatorService.maybeCreate("/p3", CreateMode.PERSISTENT, + RegistrySecurity.WorldReadWriteACL, false)); + assertFalse(curatorService.maybeCreate("/p3", CreateMode.PERSISTENT, + RegistrySecurity.WorldReadWriteACL, false)); + } + + @Test + public void testRM() throws Throwable { + mkPath("/rm", CreateMode.PERSISTENT); + curatorService.zkDelete("/rm", false, null); + verifyNotExists("/rm"); + curatorService.zkDelete("/rm", false, null); + } + + @Test + public void testRMNonRf() throws Throwable { + mkPath("/rm", CreateMode.PERSISTENT); + mkPath("/rm/child", CreateMode.PERSISTENT); + try { + curatorService.zkDelete("/rm", false, null); + fail("expected a failure"); + } catch (PathIsNotEmptyDirectoryException expected) { + + } + } + + @Test + public void testRMRf() throws Throwable { + mkPath("/rm", CreateMode.PERSISTENT); + mkPath("/rm/child", CreateMode.PERSISTENT); + curatorService.zkDelete("/rm", true, null); + verifyNotExists("/rm"); + curatorService.zkDelete("/rm", true, null); + } + + + @Test + public void testBackgroundDelete() throws Throwable { + mkPath("/rm", CreateMode.PERSISTENT); + mkPath("/rm/child", CreateMode.PERSISTENT); + CuratorEventCatcher events = new CuratorEventCatcher(); + curatorService.zkDelete("/rm", true, events); + CuratorEvent taken = events.take(); + LOG.info("took {}", taken); + assertEquals(1, events.getCount()); + } + + @Test + public void testCreate() throws Throwable { + + curatorService.zkCreate("/testcreate", + CreateMode.PERSISTENT, getTestBuffer(), + rootACL + ); + pathMustExist("/testcreate"); + } + + @Test + public void testCreateTwice() throws Throwable { + byte[] buffer = getTestBuffer(); + curatorService.zkCreate("/testcreatetwice", + CreateMode.PERSISTENT, buffer, + rootACL); + try { + curatorService.zkCreate("/testcreatetwice", + CreateMode.PERSISTENT, buffer, + rootACL); + fail(); + } catch (FileAlreadyExistsException e) { + + } + } + + @Test + public void testCreateUpdate() throws Throwable { + byte[] buffer = getTestBuffer(); + curatorService.zkCreate("/testcreateupdate", + CreateMode.PERSISTENT, buffer, + rootACL + ); + curatorService.zkUpdate("/testcreateupdate", buffer); + } + + @Test(expected = PathNotFoundException.class) + public void testUpdateMissing() throws Throwable { + curatorService.zkUpdate("/testupdatemissing", getTestBuffer()); + } + + @Test + public void testUpdateDirectory() throws Throwable { + mkPath("/testupdatedirectory", CreateMode.PERSISTENT); + curatorService.zkUpdate("/testupdatedirectory", getTestBuffer()); + } + + @Test + public void testUpdateDirectorywithChild() throws Throwable { + mkPath("/testupdatedirectorywithchild", CreateMode.PERSISTENT); + mkPath("/testupdatedirectorywithchild/child", CreateMode.PERSISTENT); + curatorService.zkUpdate("/testupdatedirectorywithchild", getTestBuffer()); + } + + @Test + public void testUseZKServiceForBinding() throws Throwable { + CuratorService cs2 = new CuratorService("curator", zookeeper); + cs2.init(new Configuration()); + cs2.start(); + } + + protected byte[] getTestBuffer() { + byte[] buffer = new byte[1]; + buffer[0] = '0'; + return buffer; + } + + + public void verifyNotExists(String path) throws IOException { + if (curatorService.zkPathExists(path)) { + fail("Path should not exist: " + path); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/TestMicroZookeeperService.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/TestMicroZookeeperService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/TestMicroZookeeperService.java new file mode 100644 index 0000000..4dfe453 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/impl/TestMicroZookeeperService.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.registry.client.impl; + +import org.apache.hadoop.service.ServiceOperations; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.registry.server.services.MicroZookeeperService; +import org.junit.After; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.rules.Timeout; + +import java.io.IOException; + +/** + * Simple tests to look at the micro ZK service itself + */ +public class TestMicroZookeeperService extends Assert { + + private MicroZookeeperService zookeeper; + + @Rule + public final Timeout testTimeout = new Timeout(10000); + @Rule + public TestName methodName = new TestName(); + + @After + public void destroyZKServer() throws IOException { + + ServiceOperations.stop(zookeeper); + } + + @Test + public void testTempDirSupport() throws Throwable { + YarnConfiguration conf = new YarnConfiguration(); + zookeeper = new MicroZookeeperService("t1"); + zookeeper.init(conf); + zookeeper.start(); + zookeeper.stop(); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java new file mode 100644 index 0000000..451a69b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestRegistryRMOperations.java @@ -0,0 +1,369 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.registry.integration; + +import org.apache.curator.framework.api.BackgroundCallback; +import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; +import org.apache.hadoop.registry.AbstractRegistryTest; +import org.apache.hadoop.registry.client.api.BindFlags; +import org.apache.hadoop.registry.client.api.RegistryConstants; +import org.apache.hadoop.registry.client.binding.RegistryUtils; +import org.apache.hadoop.registry.client.binding.RegistryPathUtils; +import org.apache.hadoop.registry.client.impl.zk.ZKPathDumper; +import org.apache.hadoop.registry.client.impl.CuratorEventCatcher; +import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; +import org.apache.hadoop.registry.client.types.RegistryPathStatus; +import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; +import org.apache.hadoop.registry.server.services.DeleteCompletionCallback; +import org.apache.hadoop.registry.server.services.RegistryAdminService; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.net.URI; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.inetAddrEndpoint; +import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.restEndpoint; + +public class TestRegistryRMOperations extends AbstractRegistryTest { + protected static final Logger LOG = + LoggerFactory.getLogger(TestRegistryRMOperations.class); + + /** + * trigger a purge operation + * @param path path + * @param id yarn ID + * @param policyMatch policy to match ID on + * @param purgePolicy policy when there are children under a match + * @return the number purged + * @throws IOException + */ + public int purge(String path, + String id, + String policyMatch, + RegistryAdminService.PurgePolicy purgePolicy) throws + IOException, + ExecutionException, + InterruptedException { + return purge(path, id, policyMatch, purgePolicy, null); + } + + /** + * + * trigger a purge operation + * @param path pathn + * @param id yarn ID + * @param policyMatch policy to match ID on + * @param purgePolicy policy when there are children under a match + * @param callback optional callback + * @return the number purged + * @throws IOException + */ + public int purge(String path, + String id, + String policyMatch, + RegistryAdminService.PurgePolicy purgePolicy, + BackgroundCallback callback) throws + IOException, + ExecutionException, + InterruptedException { + + Future<Integer> future = registry.purgeRecordsAsync(path, + id, policyMatch, purgePolicy, callback); + try { + return future.get(); + } catch (ExecutionException e) { + if (e.getCause() instanceof IOException) { + throw (IOException) e.getCause(); + } else { + throw e; + } + } + } + + @Test + public void testPurgeEntryCuratorCallback() throws Throwable { + + String path = "/users/example/hbase/hbase1/"; + ServiceRecord written = buildExampleServiceEntry( + PersistencePolicies.APPLICATION_ATTEMPT); + written.set(YarnRegistryAttributes.YARN_ID, + "testAsyncPurgeEntry_attempt_001"); + + operations.mknode(RegistryPathUtils.parentOf(path), true); + operations.bind(path, written, 0); + + ZKPathDumper dump = registry.dumpPath(false); + CuratorEventCatcher events = new CuratorEventCatcher(); + + LOG.info("Initial state {}", dump); + + // container query + String id = written.get(YarnRegistryAttributes.YARN_ID, ""); + int opcount = purge("/", + id, + PersistencePolicies.CONTAINER, + RegistryAdminService.PurgePolicy.PurgeAll, + events); + assertPathExists(path); + assertEquals(0, opcount); + assertEquals("Event counter", 0, events.getCount()); + + // now the application attempt + opcount = purge("/", + id, + PersistencePolicies.APPLICATION_ATTEMPT, + RegistryAdminService.PurgePolicy.PurgeAll, + events); + + LOG.info("Final state {}", dump); + + assertPathNotFound(path); + assertEquals("wrong no of delete operations in " + dump, 1, opcount); + // and validate the callback event + assertEquals("Event counter", 1, events.getCount()); + } + + @Test + public void testAsyncPurgeEntry() throws Throwable { + + String path = "/users/example/hbase/hbase1/"; + ServiceRecord written = buildExampleServiceEntry( + PersistencePolicies.APPLICATION_ATTEMPT); + written.set(YarnRegistryAttributes.YARN_ID, + "testAsyncPurgeEntry_attempt_001"); + + operations.mknode(RegistryPathUtils.parentOf(path), true); + operations.bind(path, written, 0); + + ZKPathDumper dump = registry.dumpPath(false); + + LOG.info("Initial state {}", dump); + + DeleteCompletionCallback deletions = new DeleteCompletionCallback(); + int opcount = purge("/", + written.get(YarnRegistryAttributes.YARN_ID, ""), + PersistencePolicies.CONTAINER, + RegistryAdminService.PurgePolicy.PurgeAll, + deletions); + assertPathExists(path); + + dump = registry.dumpPath(false); + + assertEquals("wrong no of delete operations in " + dump, 0, + deletions.getEventCount()); + assertEquals("wrong no of delete operations in " + dump, 0, opcount); + + + // now app attempt + deletions = new DeleteCompletionCallback(); + opcount = purge("/", + written.get(YarnRegistryAttributes.YARN_ID, ""), + PersistencePolicies.APPLICATION_ATTEMPT, + RegistryAdminService.PurgePolicy.PurgeAll, + deletions); + + dump = registry.dumpPath(false); + LOG.info("Final state {}", dump); + + assertPathNotFound(path); + assertEquals("wrong no of delete operations in " + dump, 1, + deletions.getEventCount()); + assertEquals("wrong no of delete operations in " + dump, 1, opcount); + // and validate the callback event + + } + + @Test + public void testPutGetContainerPersistenceServiceEntry() throws Throwable { + + String path = ENTRY_PATH; + ServiceRecord written = buildExampleServiceEntry( + PersistencePolicies.CONTAINER); + + operations.mknode(RegistryPathUtils.parentOf(path), true); + operations.bind(path, written, BindFlags.CREATE); + ServiceRecord resolved = operations.resolve(path); + validateEntry(resolved); + assertMatches(written, resolved); + } + + /** + * Create a complex example app + * @throws Throwable + */ + @Test + public void testCreateComplexApplication() throws Throwable { + String appId = "application_1408631738011_0001"; + String cid = "container_1408631738011_0001_01_"; + String cid1 = cid + "000001"; + String cid2 = cid + "000002"; + String appPath = USERPATH + "tomcat"; + + ServiceRecord webapp = createRecord(appId, + PersistencePolicies.APPLICATION, "tomcat-based web application", + null); + webapp.addExternalEndpoint(restEndpoint("www", + new URI("http", "//loadbalancer/", null))); + + ServiceRecord comp1 = createRecord(cid1, PersistencePolicies.CONTAINER, + null, + null); + comp1.addExternalEndpoint(restEndpoint("www", + new URI("http", "//rack4server3:43572", null))); + comp1.addInternalEndpoint( + inetAddrEndpoint("jmx", "JMX", "rack4server3", 43573)); + + // Component 2 has a container lifespan + ServiceRecord comp2 = createRecord(cid2, PersistencePolicies.CONTAINER, + null, + null); + comp2.addExternalEndpoint(restEndpoint("www", + new URI("http", "//rack1server28:35881", null))); + comp2.addInternalEndpoint( + inetAddrEndpoint("jmx", "JMX", "rack1server28", 35882)); + + operations.mknode(USERPATH, false); + operations.bind(appPath, webapp, BindFlags.OVERWRITE); + String componentsPath = appPath + RegistryConstants.SUBPATH_COMPONENTS; + operations.mknode(componentsPath, false); + String dns1 = RegistryPathUtils.encodeYarnID(cid1); + String dns1path = componentsPath + dns1; + operations.bind(dns1path, comp1, BindFlags.CREATE); + String dns2 = RegistryPathUtils.encodeYarnID(cid2); + String dns2path = componentsPath + dns2; + operations.bind(dns2path, comp2, BindFlags.CREATE); + + ZKPathDumper pathDumper = registry.dumpPath(false); + LOG.info(pathDumper.toString()); + + logRecord("tomcat", webapp); + logRecord(dns1, comp1); + logRecord(dns2, comp2); + + ServiceRecord dns1resolved = operations.resolve(dns1path); + assertEquals("Persistence policies on resolved entry", + PersistencePolicies.CONTAINER, + dns1resolved.get(YarnRegistryAttributes.YARN_PERSISTENCE, "")); + + Map<String, RegistryPathStatus> children = + RegistryUtils.statChildren(operations, componentsPath); + assertEquals(2, children.size()); + Collection<RegistryPathStatus> + componentStats = children.values(); + Map<String, ServiceRecord> records = + RegistryUtils.extractServiceRecords(operations, + componentsPath, componentStats); + assertEquals(2, records.size()); + ServiceRecord retrieved1 = records.get(dns1path); + logRecord(retrieved1.get(YarnRegistryAttributes.YARN_ID, ""), retrieved1); + assertMatches(dns1resolved, retrieved1); + assertEquals(PersistencePolicies.CONTAINER, + retrieved1.get(YarnRegistryAttributes.YARN_PERSISTENCE, "")); + + // create a listing under components/ + operations.mknode(componentsPath + "subdir", false); + + // this shows up in the listing of child entries + Map<String, RegistryPathStatus> childrenUpdated = + RegistryUtils.statChildren(operations, componentsPath); + assertEquals(3, childrenUpdated.size()); + + // the non-record child this is not picked up in the record listing + Map<String, ServiceRecord> recordsUpdated = + + RegistryUtils.extractServiceRecords(operations, + componentsPath, + childrenUpdated); + assertEquals(2, recordsUpdated.size()); + + // now do some deletions. + + // synchronous delete container ID 2 + + // fail if the app policy is chosen + assertEquals(0, purge("/", cid2, PersistencePolicies.APPLICATION, + RegistryAdminService.PurgePolicy.FailOnChildren)); + // succeed for container + assertEquals(1, purge("/", cid2, PersistencePolicies.CONTAINER, + RegistryAdminService.PurgePolicy.FailOnChildren)); + assertPathNotFound(dns2path); + assertPathExists(dns1path); + + // expect a skip on children to skip + assertEquals(0, + purge("/", appId, PersistencePolicies.APPLICATION, + RegistryAdminService.PurgePolicy.SkipOnChildren)); + assertPathExists(appPath); + assertPathExists(dns1path); + + // attempt to delete app with policy of fail on children + try { + int p = purge("/", + appId, + PersistencePolicies.APPLICATION, + RegistryAdminService.PurgePolicy.FailOnChildren); + fail("expected a failure, got a purge count of " + p); + } catch (PathIsNotEmptyDirectoryException expected) { + // expected + } + assertPathExists(appPath); + assertPathExists(dns1path); + + + // now trigger recursive delete + assertEquals(1, + purge("/", appId, PersistencePolicies.APPLICATION, + RegistryAdminService.PurgePolicy.PurgeAll)); + assertPathNotFound(appPath); + assertPathNotFound(dns1path); + + } + + @Test + public void testChildDeletion() throws Throwable { + ServiceRecord app = createRecord("app1", + PersistencePolicies.APPLICATION, "app", + null); + ServiceRecord container = createRecord("container1", + PersistencePolicies.CONTAINER, "container", + null); + + operations.bind("/app", app, BindFlags.OVERWRITE); + operations.bind("/app/container", container, BindFlags.OVERWRITE); + + try { + int p = purge("/", + "app1", + PersistencePolicies.APPLICATION, + RegistryAdminService.PurgePolicy.FailOnChildren); + fail("expected a failure, got a purge count of " + p); + } catch (PathIsNotEmptyDirectoryException expected) { + // expected + } + + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestYarnPolicySelector.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestYarnPolicySelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestYarnPolicySelector.java new file mode 100644 index 0000000..441b3d7 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/integration/TestYarnPolicySelector.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.registry.integration; + +import org.apache.hadoop.registry.RegistryTestHelper; +import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; +import org.apache.hadoop.registry.client.types.RegistryPathStatus; +import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.hadoop.registry.server.integration.SelectByYarnPersistence; +import org.apache.hadoop.registry.server.services.RegistryAdminService; +import org.junit.Test; + +public class TestYarnPolicySelector extends RegistryTestHelper { + + + private ServiceRecord record = createRecord("1", + PersistencePolicies.APPLICATION, "one", + null); + private RegistryPathStatus status = new RegistryPathStatus("/", 0, 0, 1); + + public void assertSelected(boolean outcome, + RegistryAdminService.NodeSelector selector) { + boolean select = selector.shouldSelect("/", status, record); + assertEquals(selector.toString(), outcome, select); + } + + @Test + public void testByContainer() throws Throwable { + assertSelected(false, + new SelectByYarnPersistence("1", + PersistencePolicies.CONTAINER)); + } + + @Test + public void testByApp() throws Throwable { + assertSelected(true, + new SelectByYarnPersistence("1", + PersistencePolicies.APPLICATION)); + } + + + @Test + public void testByAppName() throws Throwable { + assertSelected(false, + new SelectByYarnPersistence("2", + PersistencePolicies.APPLICATION)); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/operations/TestRegistryOperations.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/operations/TestRegistryOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/operations/TestRegistryOperations.java new file mode 100644 index 0000000..1cfb025 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/operations/TestRegistryOperations.java @@ -0,0 +1,304 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.registry.operations; + +import org.apache.hadoop.fs.FileAlreadyExistsException; +import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException; +import org.apache.hadoop.fs.PathNotFoundException; +import org.apache.hadoop.registry.AbstractRegistryTest; +import org.apache.hadoop.registry.client.api.BindFlags; +import org.apache.hadoop.registry.client.binding.RegistryUtils; +import org.apache.hadoop.registry.client.binding.RegistryPathUtils; +import org.apache.hadoop.registry.client.exceptions.NoRecordException; +import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; +import org.apache.hadoop.registry.client.types.RegistryPathStatus; +import org.apache.hadoop.registry.client.types.ServiceRecord; +import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class TestRegistryOperations extends AbstractRegistryTest { + protected static final Logger LOG = + LoggerFactory.getLogger(TestRegistryOperations.class); + + @Test + public void testPutGetServiceEntry() throws Throwable { + ServiceRecord written = putExampleServiceEntry(ENTRY_PATH, 0, + PersistencePolicies.APPLICATION); + ServiceRecord resolved = operations.resolve(ENTRY_PATH); + validateEntry(resolved); + assertMatches(written, resolved); + } + + @Test + public void testDeleteServiceEntry() throws Throwable { + putExampleServiceEntry(ENTRY_PATH, 0); + operations.delete(ENTRY_PATH, false); + } + + @Test + public void testDeleteNonexistentEntry() throws Throwable { + operations.delete(ENTRY_PATH, false); + operations.delete(ENTRY_PATH, true); + } + + @Test + public void testStat() throws Throwable { + putExampleServiceEntry(ENTRY_PATH, 0); + RegistryPathStatus stat = operations.stat(ENTRY_PATH); + assertTrue(stat.size > 0); + assertTrue(stat.time > 0); + assertEquals(NAME, stat.path); + } + + @Test + public void testLsParent() throws Throwable { + ServiceRecord written = putExampleServiceEntry(ENTRY_PATH, 0); + RegistryPathStatus stat = operations.stat(ENTRY_PATH); + + List<String> children = operations.list(PARENT_PATH); + assertEquals(1, children.size()); + assertEquals(NAME, children.get(0)); + Map<String, RegistryPathStatus> childStats = + RegistryUtils.statChildren(operations, PARENT_PATH); + assertEquals(1, childStats.size()); + assertEquals(stat, childStats.get(NAME)); + + Map<String, ServiceRecord> records = + RegistryUtils.extractServiceRecords(operations, + PARENT_PATH, + childStats.values()); + assertEquals(1, records.size()); + ServiceRecord record = records.get(ENTRY_PATH); + assertNotNull(record); + record.validate(); + assertMatches(written, record); + + } + + @Test + public void testDeleteNonEmpty() throws Throwable { + putExampleServiceEntry(ENTRY_PATH, 0); + try { + operations.delete(PARENT_PATH, false); + fail("Expected a failure"); + } catch (PathIsNotEmptyDirectoryException expected) { + // expected; ignore + } + operations.delete(PARENT_PATH, true); + } + + @Test(expected = PathNotFoundException.class) + public void testStatEmptyPath() throws Throwable { + operations.stat(ENTRY_PATH); + } + + @Test(expected = PathNotFoundException.class) + public void testLsEmptyPath() throws Throwable { + operations.list(PARENT_PATH); + } + + @Test(expected = PathNotFoundException.class) + public void testResolveEmptyPath() throws Throwable { + operations.resolve(ENTRY_PATH); + } + + @Test + public void testMkdirNoParent() throws Throwable { + String path = ENTRY_PATH + "/missing"; + try { + operations.mknode(path, false); + RegistryPathStatus stat = operations.stat(path); + fail("Got a status " + stat); + } catch (PathNotFoundException expected) { + // expected + } + } + + @Test + public void testDoubleMkdir() throws Throwable { + operations.mknode(USERPATH, false); + String path = USERPATH + "newentry"; + assertTrue(operations.mknode(path, false)); + operations.stat(path); + assertFalse(operations.mknode(path, false)); + } + + @Test + public void testPutNoParent() throws Throwable { + ServiceRecord record = new ServiceRecord(); + record.set(YarnRegistryAttributes.YARN_ID, "testPutNoParent"); + String path = "/path/without/parent"; + try { + operations.bind(path, record, 0); + // didn't get a failure + // trouble + RegistryPathStatus stat = operations.stat(path); + fail("Got a status " + stat); + } catch (PathNotFoundException expected) { + // expected + } + } + + @Test + public void testPutMinimalRecord() throws Throwable { + String path = "/path/with/minimal"; + operations.mknode(path, true); + ServiceRecord record = new ServiceRecord(); + operations.bind(path, record, BindFlags.OVERWRITE); + ServiceRecord resolve = operations.resolve(path); + assertMatches(record, resolve); + + } + + @Test(expected = PathNotFoundException.class) + public void testPutNoParent2() throws Throwable { + ServiceRecord record = new ServiceRecord(); + record.set(YarnRegistryAttributes.YARN_ID, "testPutNoParent"); + String path = "/path/without/parent"; + operations.bind(path, record, 0); + } + + @Test + public void testStatDirectory() throws Throwable { + String empty = "/empty"; + operations.mknode(empty, false); + operations.stat(empty); + } + + @Test + public void testStatRootPath() throws Throwable { + operations.mknode("/", false); + operations.stat("/"); + operations.list("/"); + operations.list("/"); + } + + @Test + public void testStatOneLevelDown() throws Throwable { + operations.mknode("/subdir", true); + operations.stat("/subdir"); + } + + @Test + public void testLsRootPath() throws Throwable { + String empty = "/"; + operations.mknode(empty, false); + operations.stat(empty); + } + + @Test + public void testResolvePathThatHasNoEntry() throws Throwable { + String empty = "/empty2"; + operations.mknode(empty, false); + try { + ServiceRecord record = operations.resolve(empty); + fail("expected an exception, got " + record); + } catch (NoRecordException expected) { + // expected + } + } + + @Test + public void testOverwrite() throws Throwable { + ServiceRecord written = putExampleServiceEntry(ENTRY_PATH, 0); + ServiceRecord resolved1 = operations.resolve(ENTRY_PATH); + resolved1.description = "resolved1"; + try { + operations.bind(ENTRY_PATH, resolved1, 0); + fail("overwrite succeeded when it should have failed"); + } catch (FileAlreadyExistsException expected) { + // expected + } + + // verify there's no changed + ServiceRecord resolved2 = operations.resolve(ENTRY_PATH); + assertMatches(written, resolved2); + operations.bind(ENTRY_PATH, resolved1, BindFlags.OVERWRITE); + ServiceRecord resolved3 = operations.resolve(ENTRY_PATH); + assertMatches(resolved1, resolved3); + } + + @Test + public void testPutGetContainerPersistenceServiceEntry() throws Throwable { + + String path = ENTRY_PATH; + ServiceRecord written = buildExampleServiceEntry( + PersistencePolicies.CONTAINER); + + operations.mknode(RegistryPathUtils.parentOf(path), true); + operations.bind(path, written, BindFlags.CREATE); + ServiceRecord resolved = operations.resolve(path); + validateEntry(resolved); + assertMatches(written, resolved); + } + + @Test + public void testAddingWriteAccessIsNoOpEntry() throws Throwable { + + assertFalse(operations.addWriteAccessor("id","pass")); + operations.clearWriteAccessors(); + } + + @Test + public void testListListFully() throws Throwable { + ServiceRecord r1 = new ServiceRecord(); + ServiceRecord r2 = createRecord("i", + PersistencePolicies.PERMANENT, "r2"); + + String path = USERPATH + SC_HADOOP + "/listing" ; + operations.mknode(path, true); + String r1path = path + "/r1"; + operations.bind(r1path, r1, 0); + String r2path = path + "/r2"; + operations.bind(r2path, r2, 0); + + RegistryPathStatus r1stat = operations.stat(r1path); + assertEquals("r1", r1stat.path); + RegistryPathStatus r2stat = operations.stat(r2path); + assertEquals("r2", r2stat.path); + assertNotEquals(r1stat, r2stat); + + // listings now + List<String> list = operations.list(path); + assertEquals("Wrong no. of children", 2, list.size()); + // there's no order here, so create one + Map<String, String> names = new HashMap<String, String>(); + String entries = ""; + for (String child : list) { + names.put(child, child); + entries += child + " "; + } + assertTrue("No 'r1' in " + entries, + names.containsKey("r1")); + assertTrue("No 'r2' in " + entries, + names.containsKey("r2")); + + Map<String, RegistryPathStatus> stats = + RegistryUtils.statChildren(operations, path); + assertEquals("Wrong no. of children", 2, stats.size()); + assertEquals(r1stat, stats.get("r1")); + assertEquals(r2stat, stats.get("r2")); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java new file mode 100644 index 0000000..ca3f9c9 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java @@ -0,0 +1,356 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.registry.secure; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.minikdc.MiniKdc; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.hadoop.service.Service; +import org.apache.hadoop.service.ServiceOperations; +import org.apache.hadoop.registry.RegistryTestHelper; +import org.apache.hadoop.registry.client.impl.zk.RegistrySecurity; +import org.apache.hadoop.registry.client.impl.zk.ZookeeperConfigOptions; +import org.apache.hadoop.registry.server.services.AddingCompositeService; +import org.apache.hadoop.registry.server.services.MicroZookeeperService; +import org.apache.hadoop.registry.server.services.MicroZookeeperServiceKeys; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.rules.TestName; +import org.junit.rules.Timeout; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.security.auth.Subject; +import javax.security.auth.kerberos.KerberosPrincipal; +import javax.security.auth.login.LoginContext; +import javax.security.auth.login.LoginException; +import java.io.File; +import java.io.IOException; +import java.security.Principal; +import java.util.HashSet; +import java.util.Properties; +import java.util.Set; + +/** + * Add kerberos tests. This is based on the (JUnit3) KerberosSecurityTestcase + * and its test case, <code>TestMiniKdc</code> + */ +public class AbstractSecureRegistryTest extends RegistryTestHelper { + public static final String REALM = "EXAMPLE.COM"; + public static final String ZOOKEEPER = "zookeeper"; + public static final String ZOOKEEPER_LOCALHOST = "zookeeper/localhost"; + public static final String ZOOKEEPER_REALM = "zookeeper@" + REALM; + public static final String ZOOKEEPER_CLIENT_CONTEXT = ZOOKEEPER; + public static final String ZOOKEEPER_SERVER_CONTEXT = "ZOOKEEPER_SERVER"; + ; + public static final String ZOOKEEPER_LOCALHOST_REALM = + ZOOKEEPER_LOCALHOST + "@" + REALM; + public static final String ALICE = "alice"; + public static final String ALICE_CLIENT_CONTEXT = "alice"; + public static final String ALICE_LOCALHOST = "alice/localhost"; + public static final String BOB = "bob"; + public static final String BOB_CLIENT_CONTEXT = "bob"; + public static final String BOB_LOCALHOST = "bob/localhost"; + + + private static final Logger LOG = + LoggerFactory.getLogger(AbstractSecureRegistryTest.class); + + public static final Configuration CONF; + + static { + CONF = new Configuration(); + CONF.set("hadoop.security.authentication", "kerberos"); + CONF.setBoolean("hadoop.security.authorization", true); + } + + private static final AddingCompositeService classTeardown = + new AddingCompositeService("classTeardown"); + + // static initializer guarantees it is always started + // ahead of any @BeforeClass methods + static { + classTeardown.init(CONF); + classTeardown.start(); + } + + public static final String SUN_SECURITY_KRB5_DEBUG = + "sun.security.krb5.debug"; + + private final AddingCompositeService teardown = + new AddingCompositeService("teardown"); + + protected static MiniKdc kdc; + protected static File keytab_zk; + protected static File keytab_bob; + protected static File keytab_alice; + protected static File kdcWorkDir; + protected static Properties kdcConf; + protected static RegistrySecurity registrySecurity; + + @Rule + public final Timeout testTimeout = new Timeout(900000); + + @Rule + public TestName methodName = new TestName(); + protected MicroZookeeperService secureZK; + protected static File jaasFile; + private LoginContext zookeeperLogin; + + /** + * All class initialization for this test class + * @throws Exception + */ + @BeforeClass + public static void beforeSecureRegistryTestClass() throws Exception { + registrySecurity = new RegistrySecurity("registrySecurity"); + registrySecurity.init(CONF); + setupKDCAndPrincipals(); + RegistrySecurity.clearJaasSystemProperties(); + RegistrySecurity.bindJVMtoJAASFile(jaasFile); + initHadoopSecurity(); + } + + @AfterClass + public static void afterSecureRegistryTestClass() throws + Exception { + describe(LOG, "teardown of class"); + classTeardown.close(); + teardownKDC(); + } + + /** + * give our thread a name + */ + @Before + public void nameThread() { + Thread.currentThread().setName("JUnit"); + } + + /** + * For unknown reasons, the before-class setting of the JVM properties were + * not being picked up. This method addresses that by setting them + * before every test case + */ + @Before + public void beforeSecureRegistryTest() { + + } + + @After + public void afterSecureRegistryTest() throws IOException { + describe(LOG, "teardown of instance"); + teardown.close(); + stopSecureZK(); + } + + protected static void addToClassTeardown(Service svc) { + classTeardown.addService(svc); + } + + protected void addToTeardown(Service svc) { + teardown.addService(svc); + } + + + public static void teardownKDC() throws Exception { + if (kdc != null) { + kdc.stop(); + kdc = null; + } + } + + /** + * Sets up the KDC and a set of principals in the JAAS file + * + * @throws Exception + */ + public static void setupKDCAndPrincipals() throws Exception { + // set up the KDC + File target = new File(System.getProperty("test.dir", "target")); + kdcWorkDir = new File(target, "kdc"); + kdcWorkDir.mkdirs(); + if (!kdcWorkDir.mkdirs()) { + assertTrue(kdcWorkDir.isDirectory()); + } + kdcConf = MiniKdc.createConf(); + kdcConf.setProperty(MiniKdc.DEBUG, "true"); + kdc = new MiniKdc(kdcConf, kdcWorkDir); + kdc.start(); + + keytab_zk = createKeytab(ZOOKEEPER, "zookeeper.keytab"); + keytab_alice = createKeytab(ALICE, "alice.keytab"); + keytab_bob = createKeytab(BOB, "bob.keytab"); + + StringBuilder jaas = new StringBuilder(1024); + jaas.append(registrySecurity.createJAASEntry(ZOOKEEPER_CLIENT_CONTEXT, + ZOOKEEPER, keytab_zk)); + jaas.append(registrySecurity.createJAASEntry(ZOOKEEPER_SERVER_CONTEXT, + ZOOKEEPER_LOCALHOST, keytab_zk)); + jaas.append(registrySecurity.createJAASEntry(ALICE_CLIENT_CONTEXT, + ALICE_LOCALHOST , keytab_alice)); + jaas.append(registrySecurity.createJAASEntry(BOB_CLIENT_CONTEXT, + BOB_LOCALHOST, keytab_bob)); + + jaasFile = new File(kdcWorkDir, "jaas.txt"); + FileUtils.write(jaasFile, jaas.toString()); + LOG.info("\n"+ jaas); + RegistrySecurity.bindJVMtoJAASFile(jaasFile); + } + + + // + protected static final String kerberosRule = + "RULE:[1:$1@$0](.*@EXAMPLE.COM)s/@.*//\nDEFAULT"; + + /** + * Init hadoop security by setting up the UGI config + */ + public static void initHadoopSecurity() { + + UserGroupInformation.setConfiguration(CONF); + + KerberosName.setRules(kerberosRule); + } + + /** + * Stop the secure ZK and log out the ZK account + */ + public synchronized void stopSecureZK() { + ServiceOperations.stop(secureZK); + secureZK = null; + logout(zookeeperLogin); + zookeeperLogin = null; + } + + + public static MiniKdc getKdc() { + return kdc; + } + + public static File getKdcWorkDir() { + return kdcWorkDir; + } + + public static Properties getKdcConf() { + return kdcConf; + } + + /** + * Create a secure instance + * @param name instance name + * @return the instance + * @throws Exception + */ + protected static MicroZookeeperService createSecureZKInstance(String name) + throws Exception { + String context = ZOOKEEPER_SERVER_CONTEXT; + Configuration conf = new Configuration(); + + File testdir = new File(System.getProperty("test.dir", "target")); + File workDir = new File(testdir, name); + if (!workDir.mkdirs()) { + assertTrue(workDir.isDirectory()); + } + System.setProperty( + ZookeeperConfigOptions.PROP_ZK_SERVER_MAINTAIN_CONNECTION_DESPITE_SASL_FAILURE, + "false"); + RegistrySecurity.validateContext(context); + conf.set(MicroZookeeperServiceKeys.KEY_REGISTRY_ZKSERVICE_JAAS_CONTEXT, + context); + MicroZookeeperService secureZK = new MicroZookeeperService(name); + secureZK.init(conf); + LOG.info(secureZK.getDiagnostics()); + return secureZK; + } + + /** + * Create the keytabl for the given principal, includes + * raw principal and $principal/localhost + * @param principal principal short name + * @param filename filename of keytab + * @return file of keytab + * @throws Exception + */ + public static File createKeytab(String principal, + String filename) throws Exception { + assertNotEmpty("empty principal", principal); + assertNotEmpty("empty host", filename); + assertNotNull("Null KDC", kdc); + File keytab = new File(kdcWorkDir, filename); + kdc.createPrincipal(keytab, principal, principal +"/localhost"); + return keytab; + } + + public static String getPrincipalAndRealm(String principal) { + return principal + "@" + getRealm(); + } + + protected static String getRealm() { + return kdc.getRealm(); + } + + + /** + * Log in, defaulting to the client context + * @param principal principal + * @param context context + * @param keytab keytab + * @return the logged in context + * @throws LoginException failure to log in + */ + protected LoginContext login(String principal, + String context, File keytab) throws LoginException { + LOG.info("Logging in as {} in context {} with keytab {}", + principal, context, keytab); + Set<Principal> principals = new HashSet<Principal>(); + principals.add(new KerberosPrincipal(principal)); + Subject subject = new Subject(false, principals, new HashSet<Object>(), + new HashSet<Object>()); + LoginContext login; + login = new LoginContext(context, subject, null, + KerberosConfiguration.createClientConfig(principal, keytab)); + login.login(); + return login; + } + + + /** + * Start the secure ZK instance using the test method name as the path. + * As the entry is saved to the {@link #secureZK} field, it + * is automatically stopped after the test case. + * @throws Exception on any failure + */ + protected synchronized void startSecureZK() throws Exception { + assertNull("Zookeeper is already running", secureZK); + + zookeeperLogin = login(ZOOKEEPER_LOCALHOST, + ZOOKEEPER_SERVER_CONTEXT, + keytab_zk); + secureZK = createSecureZKInstance("test-" + methodName.getMethodName()); + secureZK.start(); + } + + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/KerberosConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/KerberosConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/KerberosConfiguration.java new file mode 100644 index 0000000..f511bf6 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/KerberosConfiguration.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.registry.secure; + +import org.apache.hadoop.security.authentication.util.KerberosUtil; + +import javax.security.auth.login.AppConfigurationEntry; +import java.io.File; +import java.util.HashMap; +import java.util.Map; + +class KerberosConfiguration extends javax.security.auth.login.Configuration { + private String principal; + private String keytab; + private boolean isInitiator; + + KerberosConfiguration(String principal, File keytab, + boolean client) { + this.principal = principal; + this.keytab = keytab.getAbsolutePath(); + this.isInitiator = client; + } + + public static javax.security.auth.login.Configuration createClientConfig( + String principal, + File keytab) { + return new KerberosConfiguration(principal, keytab, true); + } + + public static javax.security.auth.login.Configuration createServerConfig( + String principal, + File keytab) { + return new KerberosConfiguration(principal, keytab, false); + } + + @Override + public AppConfigurationEntry[] getAppConfigurationEntry(String name) { + Map<String, String> options = new HashMap<String, String>(); + options.put("keyTab", keytab); + options.put("principal", principal); + options.put("useKeyTab", "true"); + options.put("storeKey", "true"); + options.put("doNotPrompt", "true"); + options.put("useTicketCache", "true"); + options.put("renewTGT", "true"); + options.put("refreshKrb5Config", "true"); + options.put("isInitiator", Boolean.toString(isInitiator)); + String ticketCache = System.getenv("KRB5CCNAME"); + if (ticketCache != null) { + options.put("ticketCache", ticketCache); + } + options.put("debug", "true"); + + return new AppConfigurationEntry[]{ + new AppConfigurationEntry(KerberosUtil.getKrb5LoginModuleName(), + AppConfigurationEntry.LoginModuleControlFlag.REQUIRED, + options) + }; + } + + @Override + public String toString() { + return "KerberosConfiguration with principal " + principal; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestRegistrySecurityHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestRegistrySecurityHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestRegistrySecurityHelper.java new file mode 100644 index 0000000..8d0dc6a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestRegistrySecurityHelper.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.registry.secure; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.registry.client.api.RegistryConstants; +import org.apache.hadoop.registry.client.impl.zk.RegistrySecurity; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.data.ACL; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; + +import static org.apache.hadoop.registry.client.api.RegistryConstants.*; + +/** + * Test for registry security operations + */ +public class TestRegistrySecurityHelper extends Assert { + private static final Logger LOG = + LoggerFactory.getLogger(TestRegistrySecurityHelper.class); + + public static final String YARN_EXAMPLE_COM = "y...@example.com"; + public static final String SASL_YARN_EXAMPLE_COM = + "sasl:" + YARN_EXAMPLE_COM; + public static final String MAPRED_EXAMPLE_COM = "map...@example.com"; + public static final String SASL_MAPRED_EXAMPLE_COM = + "sasl:" + MAPRED_EXAMPLE_COM; + public static final String SASL_MAPRED_APACHE = "sasl:mapred@APACHE"; + public static final String DIGEST_F0AF = "digest:f0afbeeb00baa"; + public static final String SASL_YARN_SHORT = "sasl:yarn@"; + public static final String SASL_MAPRED_SHORT = "sasl:mapred@"; + public static final String REALM_EXAMPLE_COM = "example.com"; + private static RegistrySecurity registrySecurity; + + @BeforeClass + public static void setupTestRegistrySecurityHelper() throws IOException { + Configuration conf = new Configuration(); + conf.setBoolean(KEY_REGISTRY_SECURE, true); + conf.set(KEY_REGISTRY_KERBEROS_REALM, "KERBEROS"); + registrySecurity = new RegistrySecurity(""); + // init the ACLs OUTSIDE A KERBEROS CLUSTER + registrySecurity.init(conf); + } + + @Test + public void testACLSplitRealmed() throws Throwable { + List<String> pairs = + registrySecurity.splitAclPairs( + SASL_YARN_EXAMPLE_COM + + ", " + + SASL_MAPRED_EXAMPLE_COM, + ""); + + assertEquals(SASL_YARN_EXAMPLE_COM, pairs.get(0)); + assertEquals(SASL_MAPRED_EXAMPLE_COM, pairs.get(1)); + } + + + @Test + public void testBuildAclsRealmed() throws Throwable { + List<ACL> acls = registrySecurity.buildACLs( + SASL_YARN_EXAMPLE_COM + + ", " + + SASL_MAPRED_EXAMPLE_COM, + "", + ZooDefs.Perms.ALL); + assertEquals(YARN_EXAMPLE_COM, acls.get(0).getId().getId()); + assertEquals(MAPRED_EXAMPLE_COM, acls.get(1).getId().getId()); + } + + @Test + public void testACLDefaultRealm() throws Throwable { + List<String> pairs = + registrySecurity.splitAclPairs( + SASL_YARN_SHORT + + ", " + + SASL_MAPRED_SHORT, + REALM_EXAMPLE_COM); + + assertEquals(SASL_YARN_EXAMPLE_COM, pairs.get(0)); + assertEquals(SASL_MAPRED_EXAMPLE_COM, pairs.get(1)); + } + + @Test + public void testBuildAclsDefaultRealm() throws Throwable { + List<ACL> acls = registrySecurity.buildACLs( + SASL_YARN_SHORT + + ", " + + SASL_MAPRED_SHORT, + REALM_EXAMPLE_COM, ZooDefs.Perms.ALL); + + assertEquals(YARN_EXAMPLE_COM, acls.get(0).getId().getId()); + assertEquals(MAPRED_EXAMPLE_COM, acls.get(1).getId().getId()); + } + + @Test + public void testACLSplitNullRealm() throws Throwable { + List<String> pairs = + registrySecurity.splitAclPairs( + SASL_YARN_SHORT + + ", " + + SASL_MAPRED_SHORT, + ""); + + assertEquals(SASL_YARN_SHORT, pairs.get(0)); + assertEquals(SASL_MAPRED_SHORT, pairs.get(1)); + } + + @Test(expected = IllegalArgumentException.class) + public void testBuildAclsNullRealm() throws Throwable { + registrySecurity.buildACLs( + SASL_YARN_SHORT + + ", " + + SASL_MAPRED_SHORT, + "", ZooDefs.Perms.ALL); + fail(""); + + } + + @Test + public void testACLDefaultRealmOnlySASL() throws Throwable { + List<String> pairs = + registrySecurity.splitAclPairs( + SASL_YARN_SHORT + + ", " + + DIGEST_F0AF, + REALM_EXAMPLE_COM); + + assertEquals(SASL_YARN_EXAMPLE_COM, pairs.get(0)); + assertEquals(DIGEST_F0AF, pairs.get(1)); + } + + @Test + public void testACLSplitMixed() throws Throwable { + List<String> pairs = + registrySecurity.splitAclPairs( + SASL_YARN_SHORT + + ", " + + SASL_MAPRED_APACHE + + ", ,," + + DIGEST_F0AF, + REALM_EXAMPLE_COM); + + assertEquals(SASL_YARN_EXAMPLE_COM, pairs.get(0)); + assertEquals(SASL_MAPRED_APACHE, pairs.get(1)); + assertEquals(DIGEST_F0AF, pairs.get(2)); + } + + @Test + public void testDefaultAClsValid() throws Throwable { + registrySecurity.buildACLs( + RegistryConstants.DEFAULT_REGISTRY_SYSTEM_ACCOUNTS, + REALM_EXAMPLE_COM, ZooDefs.Perms.ALL); + } + + @Test + public void testDefaultRealm() throws Throwable { + String realm = RegistrySecurity.getDefaultRealmInJVM(); + LOG.info("Realm {}", realm); + } + + @Test + public void testUGIProperties() throws Throwable { + UserGroupInformation user = UserGroupInformation.getCurrentUser(); + ACL acl = registrySecurity.createACLForUser(user, ZooDefs.Perms.ALL); + assertFalse(RegistrySecurity.ALL_READWRITE_ACCESS.equals(acl)); + LOG.info("User {} has ACL {}", user, acl); + } + + + @Test + public void testSecurityImpliesKerberos() throws Throwable { + Configuration conf = new Configuration(); + conf.setBoolean("hadoop.security.authentication", true); + conf.setBoolean(KEY_REGISTRY_SECURE, true); + conf.set(KEY_REGISTRY_KERBEROS_REALM, "KERBEROS"); + RegistrySecurity security = new RegistrySecurity("registry security"); + try { + security.init(conf); + } catch (Exception e) { + assertTrue( + "did not find "+ RegistrySecurity.E_NO_KERBEROS + " in " + e, + e.toString().contains(RegistrySecurity.E_NO_KERBEROS)); + } + } + + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/088ae9c5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java new file mode 100644 index 0000000..ab9d490 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.registry.secure; + + + +import com.sun.security.auth.module.Krb5LoginModule; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.security.HadoopKerberosName; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.security.authentication.util.KerberosName; +import org.apache.hadoop.security.authentication.util.KerberosUtil; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.registry.client.impl.zk.RegistrySecurity; +import org.apache.hadoop.registry.client.impl.zk.ZookeeperConfigOptions; +import org.apache.zookeeper.Environment; +import org.apache.zookeeper.data.ACL; +import org.junit.Assume; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.security.auth.Subject; +import javax.security.auth.kerberos.KerberosPrincipal; +import javax.security.auth.login.LoginContext; +import javax.security.auth.login.LoginException; +import java.io.File; +import java.io.IOException; +import java.security.Principal; +import java.security.PrivilegedExceptionAction; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Verify that logins work + */ +public class TestSecureLogins extends AbstractSecureRegistryTest { + private static final Logger LOG = + LoggerFactory.getLogger(TestSecureLogins.class); + + @Test + public void testZKinKeytab() throws Throwable { + Assume.assumeTrue(!Shell.WINDOWS); + try { + String listing = ktList(keytab_zk); + assertTrue("no " + ZOOKEEPER_LOCALHOST + " in " + listing, + listing.contains(ZOOKEEPER_LOCALHOST)); + } catch (IOException e) { + LOG.debug(KTUTIL + " failure: {}", e, e); + Assume.assumeTrue("Failed to run "+ KTUTIL+": " + e, false ); + } + } + + @Test + public void testHasRealm() throws Throwable { + assertNotNull(getRealm()); + LOG.info("ZK principal = {}", getPrincipalAndRealm(ZOOKEEPER_LOCALHOST)); + } + + @Test + public void testJaasFileSetup() throws Throwable { + // the JVM has seemed inconsistent on setting up here + assertNotNull("jaasFile", jaasFile); + String confFilename = System.getProperty(Environment.JAAS_CONF_KEY); + assertEquals(jaasFile.getAbsolutePath(), confFilename); + } + + @Test + public void testJaasFileBinding() throws Throwable { + // the JVM has seemed inconsistent on setting up here + assertNotNull("jaasFile", jaasFile); + RegistrySecurity.bindJVMtoJAASFile(jaasFile); + String confFilename = System.getProperty(Environment.JAAS_CONF_KEY); + assertEquals(jaasFile.getAbsolutePath(), confFilename); + } + + + @Test + public void testClientLogin() throws Throwable { + LoginContext client = login(ALICE_LOCALHOST, + ALICE_CLIENT_CONTEXT, + keytab_alice); + + logLoginDetails(ALICE_LOCALHOST, client); + String confFilename = System.getProperty(Environment.JAAS_CONF_KEY); + assertNotNull("Unset: "+ Environment.JAAS_CONF_KEY, confFilename); + String config = FileUtils.readFileToString(new File(confFilename)); + LOG.info("{}=\n{}", confFilename, config); + RegistrySecurity.setZKSaslClientProperties(ALICE, ALICE_CLIENT_CONTEXT); + client.logout(); + } + + + @Test + public void testServerLogin() throws Throwable { + LoginContext loginContext = createLoginContextZookeeperLocalhost(); + loginContext.login(); + loginContext.logout(); + } + + public LoginContext createLoginContextZookeeperLocalhost() throws + LoginException { + String principalAndRealm = getPrincipalAndRealm(ZOOKEEPER_LOCALHOST); + Set<Principal> principals = new HashSet<Principal>(); + principals.add(new KerberosPrincipal(ZOOKEEPER_LOCALHOST)); + Subject subject = new Subject(false, principals, new HashSet<Object>(), + new HashSet<Object>()); + return new LoginContext("", subject, null, + KerberosConfiguration.createServerConfig(ZOOKEEPER_LOCALHOST, keytab_zk)); + } + + + @Test + public void testKerberosAuth() throws Throwable { + File krb5conf = getKdc().getKrb5conf(); + String krbConfig = FileUtils.readFileToString(krb5conf); + LOG.info("krb5.conf at {}:\n{}", krb5conf, krbConfig); + Subject subject = new Subject(); + + final Krb5LoginModule krb5LoginModule = new Krb5LoginModule(); + final Map<String, String> options = new HashMap<String, String>(); + options.put("keyTab", keytab_alice.getAbsolutePath()); + options.put("principal", ALICE_LOCALHOST); + options.put("debug", "true"); + options.put("doNotPrompt", "true"); + options.put("isInitiator", "true"); + options.put("refreshKrb5Config", "true"); + options.put("renewTGT", "true"); + options.put("storeKey", "true"); + options.put("useKeyTab", "true"); + options.put("useTicketCache", "true"); + + krb5LoginModule.initialize(subject, null, + new HashMap<String, String>(), + options); + + boolean loginOk = krb5LoginModule.login(); + assertTrue("Failed to login", loginOk); + boolean commitOk = krb5LoginModule.commit(); + assertTrue("Failed to Commit", commitOk); + } + + @Test + public void testDefaultRealmValid() throws Throwable { + String defaultRealm = KerberosUtil.getDefaultRealm(); + assertNotEmpty("No default Kerberos Realm", + defaultRealm); + LOG.info("Default Realm '{}'", defaultRealm); + } + + @Test + public void testKerberosRulesValid() throws Throwable { + assertTrue("!KerberosName.hasRulesBeenSet()", + KerberosName.hasRulesBeenSet()); + String rules = KerberosName.getRules(); + assertEquals(kerberosRule, rules); + LOG.info(rules); + } + + @Test + public void testValidKerberosName() throws Throwable { + + new HadoopKerberosName(ZOOKEEPER).getShortName(); + new HadoopKerberosName(ZOOKEEPER_LOCALHOST).getShortName(); + new HadoopKerberosName(ZOOKEEPER_REALM).getShortName(); + // standard rules don't pick this up + // new HadoopKerberosName(ZOOKEEPER_LOCALHOST_REALM).getShortName(); + } + + + @Test + public void testUGILogin() throws Throwable { + + UserGroupInformation ugi = loginUGI(ZOOKEEPER, keytab_zk); + RegistrySecurity.UgiInfo ugiInfo = + new RegistrySecurity.UgiInfo(ugi); + LOG.info("logged in as: {}", ugiInfo); + assertTrue("security is not enabled: " + ugiInfo, + UserGroupInformation.isSecurityEnabled()); + assertTrue("login is keytab based: " + ugiInfo, + ugi.isFromKeytab()); + + // now we are here, build a SASL ACL + ACL acl = ugi.doAs(new PrivilegedExceptionAction<ACL>() { + @Override + public ACL run() throws Exception { + return registrySecurity.createSaslACLFromCurrentUser(0); + } + }); + assertEquals(ZOOKEEPER_REALM, acl.getId().getId()); + assertEquals(ZookeeperConfigOptions.SCHEME_SASL, acl.getId().getScheme()); + registrySecurity.addSystemACL(acl); + + } + +}