http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestExecutionEnvironment.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestExecutionEnvironment.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestExecutionEnvironment.java new file mode 100644 index 0000000..3e56a72 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestExecutionEnvironment.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.tools; + +import org.apache.slider.utils.SliderTestBase; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Test execution environment. + */ +public class TestExecutionEnvironment extends SliderTestBase { + protected static final Logger LOG = + LoggerFactory.getLogger(TestExecutionEnvironment.class); + + @Test + public void testClientEnv() throws Throwable { + SliderUtils.validateSliderClientEnvironment(LOG); + } + + @Test + public void testWinutils() throws Throwable { + SliderUtils.maybeVerifyWinUtilsValid(); + } + + @Test + public void testServerEnv() throws Throwable { + SliderUtils.validateSliderServerEnvironment(LOG, true); + } + + @Test + public void testServerEnvNoDependencies() throws Throwable { + SliderUtils.validateSliderServerEnvironment(LOG, false); + } + + @Test + public void testopenSSLEnv() throws Throwable { + SliderUtils.validateOpenSSLEnv(LOG); + } + + @Test + public void testValidatePythonEnv() throws Throwable { + SliderUtils.validatePythonEnv(LOG); + } + + @Test + public void testNativeLibs() throws Throwable { + assertNativeLibrariesPresent(); + } +}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java new file mode 100644 index 0000000..bf6ee2c --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestMiscSliderUtils.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.tools; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.slider.utils.SliderTestBase; +import org.junit.Test; + +import java.net.URI; + +/** + * Test slider utils. + */ +public class TestMiscSliderUtils extends SliderTestBase { + + + public static final String CLUSTER1 = "cluster1"; + + @Test + public void testPurgeTempDir() throws Throwable { + + Configuration configuration = new Configuration(); + FileSystem fs = FileSystem.get(new URI("file:///"), configuration); + SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration); + Path inst = sliderFileSystem.createAppInstanceTempPath(CLUSTER1, "001"); + + assertTrue(fs.exists(inst)); + sliderFileSystem.purgeAppInstanceTempFiles(CLUSTER1); + assertFalse(fs.exists(inst)); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestPortScan.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestPortScan.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestPortScan.java new file mode 100644 index 0000000..0953a8b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestPortScan.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.tools; + +import org.apache.slider.core.exceptions.BadConfigException; +import org.apache.slider.core.exceptions.SliderException; +import org.junit.Test; + +import java.net.ServerSocket; +import java.util.Arrays; +import java.util.List; + +import static org.junit.Assert.*; + +/** + * Test finding a port in a range. + */ +public class TestPortScan { + + @Test + public void testScanPorts() throws Throwable { + + ServerSocket server = new ServerSocket(0); + + try { + int serverPort = server.getLocalPort(); + assertFalse(SliderUtils.isPortAvailable(serverPort)); + int port = SliderUtils.findFreePort(serverPort, 10); + assertTrue(port > 0 && serverPort < port); + } finally { + server.close(); + } + } + + @Test + public void testRequestedPortsLogic() throws Throwable { + PortScanner portScanner = new PortScanner(); + portScanner.setPortRange("5,6,8-10, 11,14 ,20 - 22"); + List<Integer> ports = portScanner.getRemainingPortsToCheck(); + List<Integer> expectedPorts = + Arrays.asList(5, 6, 8, 9, 10, 11, 14, 20, 21, 22); + assertEquals(expectedPorts, ports); + } + + @Test + public void testRequestedPortsOutOfOrder() throws Throwable { + PortScanner portScanner = new PortScanner(); + portScanner.setPortRange("8-10,5,6, 11,20 - 22, 14 "); + List<Integer> ports = portScanner.getRemainingPortsToCheck(); + List<Integer> expectedPorts = + Arrays.asList(5, 6, 8, 9, 10, 11, 14, 20, 21, 22); + assertEquals(expectedPorts, ports); + } + + @Test + public void testFindAvailablePortInRange() throws Throwable { + ServerSocket server = new ServerSocket(0); + try { + int serverPort = server.getLocalPort(); + + PortScanner portScanner = new PortScanner(); + portScanner.setPortRange("" + (serverPort-1) + "-" + (serverPort + 3)); + int port = portScanner.getAvailablePort(); + assertNotEquals(port, serverPort); + assertTrue(port >= serverPort -1 && port <= serverPort + 3); + } finally { + server.close(); + } + } + + @Test + public void testFindAvailablePortInList() throws Throwable { + ServerSocket server = new ServerSocket(0); + try { + int serverPort = server.getLocalPort(); + + PortScanner portScanner = new PortScanner(); + portScanner.setPortRange("" + (serverPort-1) + ", " + (serverPort + 1)); + int port = portScanner.getAvailablePort(); + assertNotEquals(port, serverPort); + assertTrue(port == serverPort -1 || port == serverPort + 1); + } finally { + server.close(); + } + } + + @Test + public void testNoAvailablePorts() throws Throwable { + ServerSocket server1 = new ServerSocket(0); + ServerSocket server2 = new ServerSocket(0); + try { + int serverPort1 = server1.getLocalPort(); + int serverPort2 = server2.getLocalPort(); + + PortScanner portScanner = new PortScanner(); + portScanner.setPortRange("" + serverPort1+ ", " + serverPort2); + try { + portScanner.getAvailablePort(); + fail("expected SliderException"); + } catch (SliderException e) { + // expected + } + } finally { + server1.close(); + server2.close(); + } + } + + @Test + public void testPortRemovedFromRange() throws Throwable { + ServerSocket server = new ServerSocket(0); + try { + int serverPort = server.getLocalPort(); + + PortScanner portScanner = new PortScanner(); + portScanner.setPortRange("" + (serverPort-1) + "-" + (serverPort + 3)); + int port = portScanner.getAvailablePort(); + assertNotEquals(port, serverPort); + assertTrue(port >= serverPort -1 && port <= serverPort + 3); + assertFalse(portScanner.getRemainingPortsToCheck().contains(port)); + } finally { + server.close(); + } + } + + @Test(expected = BadConfigException.class) + public void testBadRange() throws BadConfigException { + PortScanner portScanner = new PortScanner(); + // note the em dash + portScanner.setPortRange("2000â2010"); + } + + @Test(expected = BadConfigException.class) + public void testEndBeforeStart() throws BadConfigException { + PortScanner portScanner = new PortScanner(); + portScanner.setPortRange("2001-2000"); + } + + @Test(expected = BadConfigException.class) + public void testEmptyRange() throws BadConfigException { + PortScanner portScanner = new PortScanner(); + portScanner.setPortRange(""); + } + + @Test(expected = BadConfigException.class) + public void testBlankRange() throws BadConfigException { + PortScanner portScanner = new PortScanner(); + portScanner.setPortRange(" "); + } + + @Test + public void testExtraComma() throws BadConfigException { + PortScanner portScanner = new PortScanner(); + portScanner.setPortRange("2000-2001, "); + List<Integer> ports = portScanner.getRemainingPortsToCheck(); + List<Integer> expectedPorts = Arrays.asList(2000, 2001); + assertEquals(expectedPorts, ports); + } + + @Test + public void testExtraCommas() throws BadConfigException { + PortScanner portScanner = new PortScanner(); + portScanner.setPortRange("2000-2001,, ,2003,"); + List<Integer> ports = portScanner.getRemainingPortsToCheck(); + List<Integer> expectedPorts = Arrays.asList(2000, 2001, 2003); + assertEquals(expectedPorts, ports); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderFileSystem.java new file mode 100644 index 0000000..dbb7791 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderFileSystem.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.common.tools; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.slider.common.SliderXmlConfKeys; +import org.apache.slider.utils.SliderTestBase; +import org.junit.Test; + +/** + * Test slider file system. + */ +public class TestSliderFileSystem extends SliderTestBase { + private static Configuration defaultConfiguration() { + return new Configuration(); + } + + private static Configuration createConfigurationWithKV(String key, String + value) { + Configuration conf = defaultConfiguration(); + conf.set(key, value); + return conf; + } + + @Test + public void testSliderBasePathDefaultValue() throws Throwable { + Configuration configuration = defaultConfiguration(); + FileSystem fileSystem = FileSystem.get(configuration); + + SliderFileSystem fs2 = new SliderFileSystem(fileSystem, configuration); + assertEquals(fs2.getBaseApplicationPath(), new Path(fileSystem + .getHomeDirectory(), ".slider")); + } + + @Test + public void testSliderBasePathCustomValue() throws Throwable { + Configuration configuration = createConfigurationWithKV(SliderXmlConfKeys + .KEY_SLIDER_BASE_PATH, "/slider/cluster"); + FileSystem fileSystem = FileSystem.get(configuration); + SliderFileSystem fs2 = new SliderFileSystem(fileSystem, configuration); + + assertEquals(fs2.getBaseApplicationPath(), new Path("/slider/cluster")); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderTestUtils.java new file mode 100644 index 0000000..a6e7db8 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderTestUtils.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.tools; + +import org.apache.hadoop.conf.Configuration; +import org.apache.slider.utils.SliderTestUtils; +import org.junit.Test; +import org.junit.internal.AssumptionViolatedException; + +/** + * Test slider test utils. + */ +public class TestSliderTestUtils extends SliderTestUtils { + + @Test + public void testAssumeTrue() throws Throwable { + + try { + assume(true, "true"); + } catch (AssumptionViolatedException e) { + throw new Exception(e); + } + } + + @Test + public void testAssumeFalse() throws Throwable { + + try { + assume(false, "false"); + fail("expected an exception"); + } catch (AssumptionViolatedException ignored) { + //expected + } + } + + @Test + public void testAssumeBoolOptionSetInConf() throws Throwable { + Configuration conf = new Configuration(false); + conf.set("key", "true"); + try { + assumeBoolOption(conf, "key", false); + } catch (AssumptionViolatedException e) { + throw new Exception(e); + } + } + + @Test + public void testAssumeBoolOptionUnsetInConf() throws Throwable { + Configuration conf = new Configuration(false); + try { + assumeBoolOption(conf, "key", true); + } catch (AssumptionViolatedException e) { + throw new Exception(e); + } + } + + + @Test + public void testAssumeBoolOptionFalseInConf() throws Throwable { + Configuration conf = new Configuration(false); + conf.set("key", "false"); + try { + assumeBoolOption(conf, "key", true); + fail("expected an exception"); + } catch (AssumptionViolatedException ignored) { + //expected + } + } + + @Test + public void testAssumeBoolOptionFalseUnsetInConf() throws Throwable { + Configuration conf = new Configuration(false); + try { + assumeBoolOption(conf, "key", false); + fail("expected an exception"); + } catch (AssumptionViolatedException ignored) { + //expected + } + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java index 0df6047..a525e09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestSliderUtils.java @@ -17,13 +17,9 @@ package org.apache.slider.common.tools; import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; import org.apache.hadoop.yarn.api.records.ApplicationReport; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationReportPBImpl; -import org.apache.slider.tools.TestUtility; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -33,7 +29,6 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; -import java.io.InputStream; import java.util.ArrayList; import java.util.List; @@ -45,26 +40,6 @@ public class TestSliderUtils { public TemporaryFolder folder = new TemporaryFolder(); @Test - public void testGetMetaInfoStreamFromZip() throws Exception { - String zipFileName = TestUtility.createAppPackage( - folder, - "testpkg", - "test.zip", - "target/test-classes/org/apache/slider/common/tools/test"); - Configuration configuration = new Configuration(); - FileSystem fs = FileSystem.getLocal(configuration); - log.info("fs working dir is {}", fs.getWorkingDirectory().toString()); - SliderFileSystem sliderFileSystem = new SliderFileSystem(fs, configuration); - - InputStream stream = SliderUtils.getApplicationResourceInputStream( - sliderFileSystem.getFileSystem(), - new Path(zipFileName), - "metainfo.xml"); - Assert.assertTrue(stream != null); - Assert.assertTrue(stream.available() > 0); - } - - @Test public void testTruncate() { Assert.assertEquals(SliderUtils.truncate(null, 5), null); Assert.assertEquals(SliderUtils.truncate("323", -1), "323"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestWindowsSupport.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestWindowsSupport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestWindowsSupport.java new file mode 100644 index 0000000..829b897 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestWindowsSupport.java @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.tools; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.ChecksumFileSystem; +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.service.ServiceStateException; +import org.apache.hadoop.util.Shell; +import org.apache.slider.utils.YarnMiniClusterTestBase; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.net.URI; +import java.util.Arrays; +import java.util.List; +import java.util.regex.Pattern; + +/** + * Test windows support. + */ +public class TestWindowsSupport extends YarnMiniClusterTestBase { + private static final Logger LOG = + LoggerFactory.getLogger(TestWindowsSupport.class); + + private static final Pattern HAS_DRIVE_LETTER_SPECIFIER = + Pattern.compile("^/?[a-zA-Z]:"); + public static final String WINDOWS_FILE = + "C:\\Users\\Administrator\\AppData\\Local\\Temp" + + "\\junit3180177850133852404\\testpkg\\appdef_1.zip"; + + + private static boolean hasWindowsDrive(String path) { + return HAS_DRIVE_LETTER_SPECIFIER.matcher(path).find(); + } + + private static int startPositionWithoutWindowsDrive(String path) { + if (hasWindowsDrive(path)) { + return path.charAt(0) == '/' ? 3 : 2; + } else { + return 0; + } + } + + @Test + public void testHasWindowsDrive() throws Throwable { + assertTrue(hasWindowsDrive(WINDOWS_FILE)); + } + + @Test + public void testStartPosition() throws Throwable { + assertEquals(2, startPositionWithoutWindowsDrive(WINDOWS_FILE)); + } + + @Test + public void testPathHandling() throws Throwable { + assumeWindows(); + + Path path = new Path(WINDOWS_FILE); + URI uri = path.toUri(); + // assert "file" == uri.scheme + assertNull(uri.getAuthority()); + + Configuration conf = new Configuration(); + + FileSystem localfs = FileSystem.get(uri, conf); + assertTrue(localfs instanceof ChecksumFileSystem); + try { + FileStatus stat = localfs.getFileStatus(path); + fail("expected an exception, got " + stat); + } catch (FileNotFoundException fnfe) { + // expected + } + + try { + FSDataInputStream appStream = localfs.open(path); + } catch (FileNotFoundException fnfe) { + // expected + } + } + + @Test + public void testExecNonexistentBinary() throws Throwable { + assumeWindows(); + List<String> commands = Arrays.asList("undefined-application", "--version"); + try { + exec(0, commands); + fail("expected an exception"); + } catch (ServiceStateException e) { + if (!(e.getCause() instanceof FileNotFoundException)) { + throw e; + } + } + } + @Test + public void testExecNonexistentBinary2() throws Throwable { + assumeWindows(); + assertFalse(doesAppExist(Arrays.asList("undefined-application", + "--version"))); + } + + @Test + public void testEmitKillCommand() throws Throwable { + + int result = killJavaProcesses("regionserver", 9); + // we know the exit code if there is no supported kill operation + assertTrue(getKillSupported() || result == -1); + } + + @Test + public void testHadoopHomeDefined() throws Throwable { + assumeWindows(); + String hadoopHome = Shell.getHadoopHome(); + LOG.info("HADOOP_HOME={}", hadoopHome); + } + + @Test + public void testHasWinutils() throws Throwable { + assumeWindows(); + SliderUtils.maybeVerifyWinUtilsValid(); + } + + @Test + public void testExecWinutils() throws Throwable { + assumeWindows(); + String winUtilsPath = Shell.getWinUtilsPath(); + assertTrue(SliderUtils.isSet(winUtilsPath)); + File winUtils = new File(winUtilsPath); + LOG.debug("Winutils is at {}", winUtils); + + exec(0, Arrays.asList(winUtilsPath, "systeminfo")); + } + + @Test + public void testPath() throws Throwable { + String path = extractPath(); + LOG.info("Path value = {}", path); + } + + @Test + public void testFindJavac() throws Throwable { + String name = Shell.WINDOWS ? "javac.exe" : "javac"; + assertNotNull(locateExecutable(name)); + } + + @Test + public void testHadoopDLL() throws Throwable { + assumeWindows(); + // split the path + File exepath = locateExecutable("HADOOP.DLL"); + assertNotNull(exepath); + LOG.info("Hadoop DLL at: {}", exepath); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestZKIntegration.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestZKIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestZKIntegration.java new file mode 100644 index 0000000..186123d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/common/tools/TestZKIntegration.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.common.tools; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.registry.server.services.MicroZookeeperServiceKeys; +import org.apache.slider.client.SliderClient; +import org.apache.slider.core.zk.ZKIntegration; +import org.apache.slider.utils.KeysForTests; +import org.apache.slider.utils.YarnZKMiniClusterTestBase; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +/** + * Test ZK integration. + */ +public class TestZKIntegration extends YarnZKMiniClusterTestBase implements + KeysForTests { + private static final Logger LOG = + LoggerFactory.getLogger(TestZKIntegration.class); + + public static final String USER = KeysForTests.USERNAME; + public static final int CONNECT_TIMEOUT = 5000; + private ZKIntegration zki; + + @Before + public void createCluster() { + Configuration conf = getConfiguration(); + String name = methodName.getMethodName(); + File zkdir = new File("target/zk/${name}"); + FileUtil.fullyDelete(zkdir); + conf.set(MicroZookeeperServiceKeys.KEY_ZKSERVICE_DIR, zkdir + .getAbsolutePath()); + createMicroZKCluster("-"+ name, conf); + } + + @After + public void closeZKI() throws IOException { + if (zki != null) { + zki.close(); + zki = null; + } + } + + public ZKIntegration initZKI() throws IOException, InterruptedException { + zki = createZKIntegrationInstance( + getZKBinding(), methodName.getMethodName(), true, false, + CONNECT_TIMEOUT); + return zki; + } + + @Test + public void testListUserClustersWithoutAnyClusters() throws Throwable { + assertHasZKCluster(); + initZKI(); + String userPath = ZKIntegration.mkSliderUserPath(USER); + List<String> clusters = this.zki.getClusters(); + assertTrue(SliderUtils.isEmpty(clusters)); + } + + @Test + public void testListUserClustersWithOneCluster() throws Throwable { + assertHasZKCluster(); + + initZKI(); + String userPath = ZKIntegration.mkSliderUserPath(USER); + String fullPath = zki.createPath(userPath, "/cluster-", + ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.EPHEMERAL_SEQUENTIAL); + LOG.info("Ephemeral path {}", fullPath); + List<String> clusters = zki.getClusters(); + assertEquals(1, clusters.size()); + assertTrue(fullPath.endsWith(clusters.get(0))); + } + + @Test + public void testListUserClustersWithTwoCluster() throws Throwable { + initZKI(); + String userPath = ZKIntegration.mkSliderUserPath(USER); + String c1 = createEphemeralChild(zki, userPath); + LOG.info("Ephemeral path $c1"); + String c2 = createEphemeralChild(zki, userPath); + LOG.info("Ephemeral path $c2"); + List<String> clusters = zki.getClusters(); + assertEquals(2, clusters.size()); + assertTrue((c1.endsWith(clusters.get(0)) && c2.endsWith(clusters.get(1))) || + (c1.endsWith(clusters.get(1)) && c2.endsWith(clusters.get(0)))); + } + + @Test + public void testCreateAndDeleteDefaultZKPath() throws Throwable { + MockSliderClient client = new MockSliderClient(); + + String path = client.createZookeeperNodeInner("cl1", true); + zki = client.getLastZKIntegration(); + + String zkPath = ZKIntegration.mkClusterPath(USER, "cl1"); + assertEquals("zkPath must be as expected", zkPath, + "/services/slider/users/" + USER + "/cl1"); + assertEquals(path, zkPath); + assertNull("ZKIntegration should be null.", zki); + zki = createZKIntegrationInstance(getZKBinding(), "cl1", true, false, + CONNECT_TIMEOUT); + assertFalse(zki.exists(zkPath)); + + path = client.createZookeeperNodeInner("cl1", false); + zki = client.getLastZKIntegration(); + assertNotNull(zki); + assertEquals("zkPath must be as expected", zkPath, + "/services/slider/users/" + USER + "/cl1"); + assertEquals(path, zkPath); + assertTrue(zki.exists(zkPath)); + zki.createPath(zkPath, "/cn", ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode + .PERSISTENT); + assertTrue(zki.exists(zkPath + "/cn")); + client.deleteZookeeperNode("cl1"); + assertFalse(zki.exists(zkPath)); + } + + public static String createEphemeralChild(ZKIntegration zki, String userPath) + throws KeeperException, InterruptedException { + return zki.createPath(userPath, "/cluster-", + ZooDefs.Ids.OPEN_ACL_UNSAFE, + CreateMode.EPHEMERAL_SEQUENTIAL); + } + + /** + * Test slider client that overriddes ZK client. + */ + public class MockSliderClient extends SliderClient { + private ZKIntegration zki; + + @Override + public String getUsername() { + return USER; + } + + @Override + protected ZKIntegration getZkClient(String clusterName, String user) { + try { + zki = createZKIntegrationInstance(getZKBinding(), clusterName, true, + false, CONNECT_TIMEOUT); + } catch (Exception e) { + fail("creating ZKIntergration threw an exception"); + } + return zki; + } + + @Override + public Configuration getConfig() { + return new Configuration(); + } + + public ZKIntegration getLastZKIntegration() { + return zki; + } + + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java new file mode 100644 index 0000000..f13fbcc --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/ExampleConfResources.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.conf; + +import org.apache.slider.api.resource.Application; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static org.apache.slider.utils.SliderTestUtils.JSON_SER_DESER; + +/** + * Names of the example configs. + */ +public final class ExampleConfResources { + + public static final String APP_JSON = "app.json"; + public static final String APP_RES = "app-resolved.json"; + public static final String OVERRIDE_JSON = "app-override.json"; + public static final String OVERRIDE_RES = "app-override-resolved.json"; + + public static final String PACKAGE = "/org/apache/slider/core/conf/examples/"; + + + private static final String[] ALL_EXAMPLES = {APP_JSON, APP_RES, + OVERRIDE_JSON, OVERRIDE_RES}; + + public static final List<String> ALL_EXAMPLE_RESOURCES = new ArrayList<>(); + static { + for (String example : ALL_EXAMPLES) { + ALL_EXAMPLE_RESOURCES.add(PACKAGE + example); + } + } + + private ExampleConfResources() { + } + + static Application loadResource(String name) throws IOException { + return JSON_SER_DESER.fromResource(PACKAGE + name); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java new file mode 100644 index 0000000..48b0736 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfTreeLoadExamples.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.conf; + +import org.apache.slider.api.resource.Application; +import org.apache.slider.common.tools.SliderUtils; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.util.Arrays; +import java.util.Collection; + +import static org.apache.slider.utils.SliderTestUtils.JSON_SER_DESER; + +/** + * Test loading example resources. + */ +@RunWith(value = Parameterized.class) +public class TestConfTreeLoadExamples extends Assert { + private String resource; + + public TestConfTreeLoadExamples(String resource) { + this.resource = resource; + } + + @Parameterized.Parameters + public static Collection<String[]> filenames() { + String[][] stringArray = new String[ExampleConfResources + .ALL_EXAMPLE_RESOURCES.size()][1]; + int i = 0; + for (String s : ExampleConfResources.ALL_EXAMPLE_RESOURCES) { + stringArray[i++][0] = s; + } + return Arrays.asList(stringArray); + } + + @Test + public void testLoadResource() throws Throwable { + try { + Application application = JSON_SER_DESER.fromResource(resource); + SliderUtils.resolve(application); + } catch (Exception e) { + throw new Exception("exception loading " + resource + ":" + e.toString()); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java new file mode 100644 index 0000000..285ddfa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/core/conf/TestConfigurationResolve.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.core.conf; + +import org.apache.slider.api.resource.Application; +import org.apache.slider.api.resource.Configuration; +import org.apache.slider.common.tools.SliderUtils; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.slider.api.InternalKeys.CHAOS_MONKEY_INTERVAL; +import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS; +import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS; +import static org.apache.slider.api.InternalKeys.DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES; +import static org.apache.slider.core.conf.ExampleConfResources.APP_JSON; +import static org.apache.slider.core.conf.ExampleConfResources.OVERRIDE_JSON; + +/** + * Test global configuration resolution. + */ +public class TestConfigurationResolve extends Assert { + protected static final Logger LOG = + LoggerFactory.getLogger(TestConfigurationResolve.class); + + @Test + public void testOverride() throws Throwable { + + Application orig = ExampleConfResources.loadResource(OVERRIDE_JSON); + + Configuration global = orig.getConfiguration(); + assertEquals("a", global.getProperty("g1")); + assertEquals("b", global.getProperty("g2")); + + Configuration simple = orig.getComponent("simple").getConfiguration(); + assertEquals(0, simple.getProperties().size()); + + Configuration master = orig.getComponent("master").getConfiguration(); + assertEquals("m", master.getProperty("name")); + assertEquals("overridden", master.getProperty("g1")); + + Configuration worker = orig.getComponent("worker").getConfiguration(); + LOG.info("worker = {}", worker); + assertEquals(3, worker.getProperties().size()); + + assertEquals("worker", worker.getProperty("name")); + assertEquals("overridden-by-worker", worker.getProperty("g1")); + assertNull(worker.getProperty("g2")); + assertEquals("1000", worker.getProperty("timeout")); + + // here is the resolution + SliderUtils.resolve(orig); + + global = orig.getConfiguration(); + LOG.info("global = {}", global); + assertEquals("a", global.getProperty("g1")); + assertEquals("b", global.getProperty("g2")); + + simple = orig.getComponent("simple").getConfiguration(); + assertEquals(2, simple.getProperties().size()); + assertEquals("a", simple.getProperty("g1")); + assertEquals("b", simple.getProperty("g2")); + + + master = orig.getComponent("master").getConfiguration(); + LOG.info("master = {}", master); + assertEquals(3, master.getProperties().size()); + assertEquals("m", master.getProperty("name")); + assertEquals("overridden", master.getProperty("g1")); + assertEquals("b", master.getProperty("g2")); + + worker = orig.getComponent("worker").getConfiguration(); + LOG.info("worker = {}", worker); + assertEquals(4, worker.getProperties().size()); + + assertEquals("worker", worker.getProperty("name")); + assertEquals("overridden-by-worker", worker.getProperty("g1")); + assertEquals("b", worker.getProperty("g2")); + assertEquals("1000", worker.getProperty("timeout")); + + } + + @Test + public void testTimeIntervalLoading() throws Throwable { + + Application orig = ExampleConfResources.loadResource(APP_JSON); + + Configuration conf = orig.getConfiguration(); + long s = conf.getPropertyLong( + CHAOS_MONKEY_INTERVAL + SliderUtils.SECONDS, + 0); + assertEquals(60, s); + long monkeyInterval = SliderUtils.getTimeRange(conf, + CHAOS_MONKEY_INTERVAL, + DEFAULT_CHAOS_MONKEY_INTERVAL_DAYS, + DEFAULT_CHAOS_MONKEY_INTERVAL_HOURS, + DEFAULT_CHAOS_MONKEY_INTERVAL_MINUTES, + 0); + assertEquals(60L, monkeyInterval); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestFilesystemPermissions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestFilesystemPermissions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestFilesystemPermissions.java new file mode 100644 index 0000000..fd794ea --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestFilesystemPermissions.java @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.other; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.UnsupportedFileSystemException; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.DiskChecker; +import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; +import org.apache.slider.utils.YarnMiniClusterTestBase; +import org.junit.After; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * This test class exists to look at permissions of the filesystem, especially + * that created by Mini YARN clusters. On some windows jenkins machines, + * YARN actions were failing as the directories had the wrong permissions + * (i.e. too lax) + */ +public class TestFilesystemPermissions extends YarnMiniClusterTestBase { + + private static final Logger LOG = LoggerFactory.getLogger( + TestFilesystemPermissions.class); + + private List<File> filesToDelete = new ArrayList<>(); + + @After + public void deleteFiles() { + for (File f : filesToDelete) { + FileUtil.fullyDelete(f, true); + } + } + + @Test + public void testJavaFSOperations() throws Throwable { + assertNativeLibrariesPresent(); + File subdir = testDir(); + subdir.mkdir(); + assertTrue(subdir.isDirectory()); + assertTrue(FileUtil.canRead(subdir)); + assertTrue(FileUtil.canWrite(subdir)); + assertTrue(FileUtil.canExecute(subdir)); + } + + @Test + public void testDiskCheckerOperations() throws Throwable { + assertNativeLibrariesPresent(); + File subdir = testDir(); + subdir.mkdir(); + DiskChecker checker = new DiskChecker(); + checker.checkDir(subdir); + } + + @Test + public void testDiskCheckerMkdir() throws Throwable { + assertNativeLibrariesPresent(); + File subdir = testDir(); + subdir.mkdirs(); + DiskChecker checker = new DiskChecker(); + checker.checkDir(subdir); + } + + /** + * Get a test dir for this method; one that will be deleted on teardown. + * @return a filename unique to this test method + */ + File testDir() { + File parent = new File("target/testfspermissions"); + parent.mkdir(); + File testdir = new File(parent, methodName.getMethodName()); + filesToDelete.add(testdir); + return testdir; + } + + + @Test + public void testPermsMap() throws Throwable { + File dir = testDir(); + String diruri = dir.toURI().toString(); + FileContext lfs = createLocalFS(dir, getConfiguration()); + getLocalDirsPathPermissionsMap(lfs, diruri); + } + + @Test + public void testInitLocaldir() throws Throwable { + File dir = testDir(); + String diruri = dir.toURI().toString(); + FileContext lfs = createLocalFS(dir, getConfiguration()); + initializeLocalDir(lfs, diruri); + List<String> localDirs = getInitializedLocalDirs(lfs, Arrays.asList( + diruri)); + assertEquals(1, localDirs.size()); + } + + + @Test + public void testValidateMiniclusterPerms() throws Throwable { + int numLocal = 1; + String cluster = createMiniCluster("", getConfiguration(), 1, numLocal, 1, + false); + File workDir = getMiniCluster().getTestWorkDir(); + List<File> localdirs = new ArrayList<>(); + for (File file : workDir.listFiles()) { + if (file.isDirectory() && file.getAbsolutePath().contains("-local")) { + // local dir + localdirs.add(file); + } + } + assertEquals(numLocal, localdirs.size()); + FileContext lfs = createLocalFS(workDir, getConfiguration()); + for (File file : localdirs) { + checkLocalDir(lfs, file.toURI().toString()); + } + } + + FileContext createLocalFS(File dir, Configuration conf) + throws UnsupportedFileSystemException { + return FileContext.getFileContext(dir.toURI(), conf); + } + + /** + * Extracted from ResourceLocalizationService. + * @param lfs + * @param localDir + * @return perms map + * @see ResourceLocalizationService + */ + private Map<Path, FsPermission> getLocalDirsPathPermissionsMap( + FileContext lfs, + String localDir) { + Map<Path, FsPermission> localDirPathFsPermissionsMap = new HashMap<>(); + + FsPermission defaultPermission = + FsPermission.getDirDefault().applyUMask(lfs.getUMask()); + FsPermission nmPrivatePermission = + ResourceLocalizationService.NM_PRIVATE_PERM.applyUMask(lfs.getUMask()); + + Path userDir = new Path(localDir, ContainerLocalizer.USERCACHE); + Path fileDir = new Path(localDir, ContainerLocalizer.FILECACHE); + Path sysDir = new Path( + localDir, + ResourceLocalizationService.NM_PRIVATE_DIR); + + localDirPathFsPermissionsMap.put(userDir, defaultPermission); + localDirPathFsPermissionsMap.put(fileDir, defaultPermission); + localDirPathFsPermissionsMap.put(sysDir, nmPrivatePermission); + return localDirPathFsPermissionsMap; + } + + private boolean checkLocalDir(FileContext lfs, String localDir) + throws IOException { + + Map<Path, FsPermission> pathPermissionMap = + getLocalDirsPathPermissionsMap(lfs, localDir); + + for (Map.Entry<Path, FsPermission> entry : pathPermissionMap.entrySet()) { + FileStatus status; + status = lfs.getFileStatus(entry.getKey()); + + if (!status.getPermission().equals(entry.getValue())) { + String msg = + "Permissions incorrectly set for dir " + entry.getKey() + + ", should be " + entry.getValue() + ", actual value = " + + status.getPermission(); + throw new YarnRuntimeException(msg); + } + } + return true; + } + + + private void initializeLocalDir(FileContext lfs, String localDir) + throws IOException { + + Map<Path, FsPermission> pathPermissionMap = + getLocalDirsPathPermissionsMap(lfs, localDir); + for (Map.Entry<Path, FsPermission> entry : pathPermissionMap.entrySet()) { + FileStatus status; + try { + status = lfs.getFileStatus(entry.getKey()); + } catch (FileNotFoundException fs) { + status = null; + } + + if (status == null) { + lfs.mkdir(entry.getKey(), entry.getValue(), true); + status = lfs.getFileStatus(entry.getKey()); + } + FsPermission perms = status.getPermission(); + if (!perms.equals(entry.getValue())) { + lfs.setPermission(entry.getKey(), entry.getValue()); + } + } + } + + synchronized private List<String> getInitializedLocalDirs(FileContext lfs, + List<String> dirs) throws IOException { + List<String> checkFailedDirs = new ArrayList<String>(); + for (String dir : dirs) { + try { + checkLocalDir(lfs, dir); + } catch (YarnRuntimeException e) { + checkFailedDirs.add(dir); + } + } + for (String dir : checkFailedDirs) { + LOG.info("Attempting to initialize " + dir); + initializeLocalDir(lfs, dir); + checkLocalDir(lfs, dir); + } + return dirs; + } + + + private void createDir(FileContext localFs, Path dir, FsPermission perm) + throws IOException { + if (dir == null) { + return; + } + try { + localFs.getFileStatus(dir); + } catch (FileNotFoundException e) { + createDir(localFs, dir.getParent(), perm); + localFs.mkdir(dir, perm, false); + if (!perm.equals(perm.applyUMask(localFs.getUMask()))) { + localFs.setPermission(dir, perm); + } + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestLocalDirStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestLocalDirStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestLocalDirStatus.java new file mode 100644 index 0000000..704c71e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/other/TestLocalDirStatus.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.other; + +import org.apache.slider.utils.SliderTestUtils; +import org.apache.slider.utils.TestUtility; +import org.junit.Test; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; + +/** + * This test exists to diagnose local FS permissions. + */ +public class TestLocalDirStatus extends SliderTestUtils { + + + public static final int SIZE = 0x200000; + + @Test + public void testTempDir() throws Throwable { + File tmpf = null; + try { + tmpf = File.createTempFile("testl", ".bin"); + createAndReadFile(tmpf, SIZE); + tmpf.delete(); + assertFalse(tmpf.exists()); + } finally { + if (tmpf != null) { + tmpf.delete(); + } + } + } + + @Test + public void testTargetDir() throws Throwable { + File target = target(); + File tmpf = null; + try { + tmpf = File.createTempFile("testl", ".bin", target); + createAndReadFile(tmpf, SIZE); + tmpf.delete(); + assertFalse(tmpf.exists()); + } finally { + if (tmpf != null) { + tmpf.delete(); + } + + } + } + + public File target() { + File target = new File("target").getAbsoluteFile(); + assertTrue(target.exists()); + return target; + } + + @Test + public void testRenameInTargetDir() throws Throwable { + File target = target(); + File tmpf = null; + File dst= null; + try { + tmpf = File.createTempFile("testl", ".bin", target); + dst = File.createTempFile("test-dest", ".bin", target); + createRenameAndReadFile(tmpf, dst, SIZE); + assertFalse(tmpf.exists()); + dst.delete(); + } finally { + if (tmpf != null) { + tmpf.delete(); + } + if (dst != null) { + dst.delete(); + } + } + } + + @Test + public void testRenameInTmpDir() throws Throwable { + File tmpf = null; + File dst= null; + try { + tmpf = File.createTempFile("testl", ".bin"); + dst = File.createTempFile("test-dest", ".bin"); + createRenameAndReadFile(tmpf, dst, SIZE); + assertFalse(tmpf.exists()); + dst.delete(); + } finally { + if (tmpf != null) { + tmpf.delete(); + } + if (dst != null) { + dst.delete(); + } + } + } + + protected void createAndReadFile(File path, int len) throws IOException { + byte[] dataset = TestUtility.dataset(len, 32, 128); + writeFile(path, dataset); + assertTrue(path.exists()); + assertEquals(len, path.length()); + byte[] persisted = readFile(path); + TestUtility.compareByteArrays(dataset, persisted, len); + } + + protected void createRenameAndReadFile(File src, File dst, int len) + throws IOException { + byte[] dataset = TestUtility.dataset(len, 32, 128); + writeFile(src, dataset); + assertTrue(src.exists()); + assertEquals(len, src.length()); + dst.delete(); + assertFalse(dst.exists()); + assertTrue(src.renameTo(dst)); + assertEquals(len, dst.length()); + byte[] persisted = readFile(dst); + TestUtility.compareByteArrays(dataset, persisted, len); + } + + protected void writeFile(File path, byte[] dataset) + throws IOException { + FileOutputStream out = new FileOutputStream(path); + try { + out.write(dataset); + out.flush(); + } finally { + out.close(); + } + } + + protected byte[] readFile(File path) throws IOException { + assertTrue(path.getAbsoluteFile().exists()); + assertTrue(path.getAbsoluteFile().isFile()); + int len = (int)path.length(); + byte[] dataset = new byte[len]; + FileInputStream ins = new FileInputStream(path); + try { + ins.read(dataset); + } finally { + ins.close(); + } + return dataset; + } + + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java new file mode 100644 index 0000000..11abdfe --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/providers/TestProviderFactory.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.providers; + +import org.apache.slider.providers.docker.DockerKeys; +import org.apache.slider.providers.docker.DockerProviderFactory; +import org.junit.Test; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Test provider factories. + */ +public class TestProviderFactory { + @Test + public void testLoadAgentProvider() throws Throwable { + SliderProviderFactory factory = SliderProviderFactory + .createSliderProviderFactory(DockerKeys.PROVIDER_DOCKER); + assertTrue(factory instanceof DockerProviderFactory); + } + + @Test + public void testCreateClientProvider() throws Throwable { + SliderProviderFactory factory = SliderProviderFactory + .createSliderProviderFactory(DockerKeys.PROVIDER_DOCKER); + assertNotNull(factory.createClientProvider()); + } + + @Test + public void testCreateProviderByClassname() throws Throwable { + SliderProviderFactory factory = SliderProviderFactory + .createSliderProviderFactory(DockerKeys.PROVIDER_DOCKER); + assertNotNull(factory.createServerProvider()); + assertTrue(factory instanceof DockerProviderFactory); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestConfigSetNaming.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestConfigSetNaming.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestConfigSetNaming.java new file mode 100644 index 0000000..100518e --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestConfigSetNaming.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.registry; + +import org.apache.slider.core.registry.docstore.PublishedConfigSet; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Arrays; + +/** + * Test config set name validation. + */ +public class TestConfigSetNaming { + + void assertValid(String name) { + PublishedConfigSet.validateName(name); + } + + void assertInvalid(String name) { + try { + PublishedConfigSet.validateName(name); + Assert.fail("Invalid name was unexpectedly parsed: " + name); + } catch (IllegalArgumentException expected) { + // expected + } + } + + @Test + public void testLowerCase() throws Throwable { + assertValid("abcdefghijklmnopqrstuvwxyz"); + } + + @Test + public void testUpperCaseInvalid() throws Throwable { + assertInvalid("ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + } + + @Test + public void testNumbers() throws Throwable { + assertValid("01234567890"); + } + + @Test + public void testChars() throws Throwable { + assertValid("a-_+"); + } + + @Test + public void testInvalids() throws Throwable { + for (String s : Arrays.asList( + "", + " ", + "*", + "a/b", + "b\\a", + "\"", + "'", + "\u0000", + "\u0f00", + "key.value", + "-", + "+", + "_", + "?")) { + assertInvalid(s); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestRegistryPaths.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestRegistryPaths.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestRegistryPaths.java new file mode 100644 index 0000000..5fcfd89 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/registry/TestRegistryPaths.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.registry; + +import org.apache.hadoop.registry.client.binding.RegistryUtils; +import org.apache.slider.core.registry.SliderRegistryUtils; +import org.apache.slider.utils.SliderTestUtils; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Test registry paths. + */ +public class TestRegistryPaths { + + @Test + public void testHomedirKerberos() throws Throwable { + String home = RegistryUtils.homePathForUser("[email protected]"); + try { + assertEquals("/users/hbase", home); + } catch (AssertionError e) { + SliderTestUtils.skip("homedir filtering not yet in hadoop registry " + + "module"); + } + } + + @Test + public void testHomedirKerberosHost() throws Throwable { + String home = RegistryUtils.homePathForUser("hbase/localhost@HADOOP" + + ".APACHE.ORG"); + try { + assertEquals("/users/hbase", home); + } catch (AssertionError e) { + SliderTestUtils.skip("homedir filtering not yet in hadoop registry " + + "module"); + } + } + + @Test + public void testRegistryPathForInstance() throws Throwable { + String path = SliderRegistryUtils.registryPathForInstance("instance"); + assertTrue(path.endsWith("/instance")); + } + + @Test + public void testPathResolution() throws Throwable { + String home = RegistryUtils.homePathForCurrentUser(); + assertEquals(home, SliderRegistryUtils.resolvePath("~")); + assertEquals(home +"/", SliderRegistryUtils.resolvePath("~/")); + assertEquals(home +"/something", SliderRegistryUtils.resolvePath( + "~/something")); + assertEquals("~unresolved", SliderRegistryUtils.resolvePath( + "~unresolved")); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/actions/TestActions.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/actions/TestActions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/actions/TestActions.java new file mode 100644 index 0000000..68d55aa --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/actions/TestActions.java @@ -0,0 +1,246 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.actions; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.service.ServiceOperations; +import org.apache.slider.server.appmaster.SliderAppMaster; +import org.apache.slider.server.appmaster.state.AppState; +import org.apache.slider.server.services.workflow.ServiceThreadFactory; +import org.apache.slider.server.services.workflow.WorkflowExecutorService; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +/** + * Test AM actions. + */ +public class TestActions { + protected static final Logger LOG = + LoggerFactory.getLogger(TestActions.class); + + private QueueService queues; + private WorkflowExecutorService<ExecutorService> executorService; + + + @Before + public void createService() { + queues = new QueueService(); + + Configuration conf = new Configuration(); + queues.init(conf); + + queues.start(); + + executorService = new WorkflowExecutorService<>("AmExecutor", + Executors.newCachedThreadPool( + new ServiceThreadFactory("AmExecutor", true))); + + executorService.init(conf); + executorService.start(); + } + + @After + public void destroyService() { + ServiceOperations.stop(executorService); + ServiceOperations.stop(queues); + } + + @Test + public void testBasicService() throws Throwable { + queues.start(); + } + + @Test + public void testDelayLogic() throws Throwable { + ActionNoteExecuted action = new ActionNoteExecuted("", 1000); + long now = System.currentTimeMillis(); + + long delay = action.getDelay(TimeUnit.MILLISECONDS); + assertTrue(delay >= 800); + assertTrue(delay <= 1800); + + ActionNoteExecuted a2 = new ActionNoteExecuted("a2", 10000); + assertTrue(action.compareTo(a2) < 0); + assertTrue(a2.compareTo(action) > 0); + assertEquals(0, action.compareTo(action)); + + } + + @Test + public void testActionDelayedExecutorTermination() throws Throwable { + long start = System.currentTimeMillis(); + + ActionStopQueue stopAction = new ActionStopQueue(1000); + queues.scheduledActions.add(stopAction); + queues.run(); + AsyncAction take = queues.actionQueue.take(); + assertEquals(take, stopAction); + long stop = System.currentTimeMillis(); + assertTrue(stop - start > 500); + assertTrue(stop - start < 1500); + } + + @Test + public void testImmediateQueue() throws Throwable { + ActionNoteExecuted noteExecuted = new ActionNoteExecuted("executed", 0); + queues.put(noteExecuted); + queues.put(new ActionStopQueue(0)); + QueueExecutor ex = new QueueExecutor(queues); + ex.run(); + assertTrue(queues.actionQueue.isEmpty()); + assertTrue(noteExecuted.executed.get()); + } + + @Test + public void testActionOrdering() throws Throwable { + + ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500); + ActionStopQueue stop = new ActionStopQueue(1500); + ActionNoteExecuted note2 = new ActionNoteExecuted("note2", 800); + + List<AsyncAction> actions = Arrays.asList(note1, stop, note2); + Collections.sort(actions); + assertEquals(actions.get(0), note1); + assertEquals(actions.get(1), note2); + assertEquals(actions.get(2), stop); + } + + @Test + public void testDelayedQueueWithReschedule() throws Throwable { + + ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500); + ActionStopQueue stop = new ActionStopQueue(1500); + ActionNoteExecuted note2 = new ActionNoteExecuted("note2", 800); + + assertTrue(note2.compareTo(stop) < 0); + assertTrue(note1.getNanos() < note2.getNanos()); + assertTrue(note2.getNanos() < stop.getNanos()); + queues.schedule(note1); + queues.schedule(note2); + queues.schedule(stop); + // async to sync expected to run in order + runQueuesToCompletion(); + assertTrue(note1.executed.get()); + assertTrue(note2.executed.get()); + } + + public void runQueuesToCompletion() { + queues.run(); + assertTrue(queues.scheduledActions.isEmpty()); + assertFalse(queues.actionQueue.isEmpty()); + QueueExecutor ex = new QueueExecutor(queues); + ex.run(); + // flush all stop commands from the queue + queues.flushActionQueue(ActionStopQueue.class); + + assertTrue(queues.actionQueue.isEmpty()); + } + + @Test + public void testRenewedActionFiresOnceAtLeast() throws Throwable { + ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500); + RenewingAction renewer = new RenewingAction( + note1, + 500, + 100, + TimeUnit.MILLISECONDS, + 3); + queues.schedule(renewer); + ActionStopQueue stop = new ActionStopQueue(4, TimeUnit.SECONDS); + queues.schedule(stop); + // this runs all the delayed actions FIRST, so can't be used + // to play tricks of renewing actions ahead of the stop action + runQueuesToCompletion(); + assertEquals(1, renewer.executionCount.intValue()); + assertEquals(1, note1.executionCount.intValue()); + // assert the renewed item is back in + assertTrue(queues.scheduledActions.contains(renewer)); + } + + + @Test + public void testRenewingActionOperations() throws Throwable { + ActionNoteExecuted note1 = new ActionNoteExecuted("note1", 500); + RenewingAction renewer = new RenewingAction( + note1, + 100, + 100, + TimeUnit.MILLISECONDS, + 3); + queues.renewing("note", renewer); + assertTrue(queues.removeRenewingAction("note")); + queues.stop(); + assertTrue(queues.waitForServiceToStop(10000)); + } + + /** + * Test action. + */ + public class ActionNoteExecuted extends AsyncAction { + private final AtomicBoolean executed = new AtomicBoolean(false); + private final AtomicLong executionTimeNanos = new AtomicLong(); + private final AtomicLong executionCount = new AtomicLong(); + + public ActionNoteExecuted(String text, int delay) { + super(text, delay); + } + + @Override + public void execute( + SliderAppMaster appMaster, + QueueAccess queueService, + AppState appState) throws Exception { + LOG.info("Executing {}", name); + executed.set(true); + executionTimeNanos.set(System.nanoTime()); + executionCount.incrementAndGet(); + LOG.info(this.toString()); + + synchronized (this) { + this.notify(); + } + } + + @Override + public String toString() { + return super.toString() + " executed=" + executed.get() + "; count=" + + executionCount.get() + ";"; + } + + public long getExecutionCount() { + return executionCount.get(); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/256a1597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java new file mode 100644 index 0000000..c1f2886 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/BaseMockAppStateAATest.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.appstate; + +import org.apache.slider.api.ResourceKeys; +import org.apache.slider.api.resource.Application; +import org.apache.slider.providers.PlacementPolicy; +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest; +import org.apache.slider.server.appmaster.model.mock.MockRoles; +import org.apache.slider.server.appmaster.state.RoleStatus; + +import static org.apache.slider.api.ResourceKeys.COMPONENT_PLACEMENT_POLICY; + +/** + * Class for basis of Anti-affine placement tests; sets up role2 + * for anti-affinity. + */ +public class BaseMockAppStateAATest extends BaseMockAppStateTest + implements MockRoles { + + /** Role status for the base AA role. */ + private RoleStatus aaRole; + + /** Role status for the AA role requiring a node with the gpu label. */ + private RoleStatus gpuRole; + + @Override + public Application buildApplication() { + Application application = factory.newApplication(0, 0, 0) + .name(getTestName()); + application.getComponent(ROLE1).getConfiguration().setProperty( + COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy + .ANTI_AFFINITY_REQUIRED)); + application.getComponent(ROLE1).getConfiguration().setProperty( + ResourceKeys.YARN_LABEL_EXPRESSION, LABEL_GPU); + application.getComponent(ROLE2).getConfiguration().setProperty( + COMPONENT_PLACEMENT_POLICY, Integer.toString(PlacementPolicy + .ANTI_AFFINITY_REQUIRED)); + return application; + } + + + @Override + public void setup() throws Exception { + super.setup(); + aaRole = lookupRole(ROLE2); + gpuRole = lookupRole(ROLE1); + } + + protected RoleStatus getAaRole() { + return aaRole; + } + + protected RoleStatus getGpuRole() { + return gpuRole; + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
