Repository: incubator-slider Updated Branches: refs/heads/develop 094f0795c -> 424b2a406
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java index 036d98e..10c497f 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java @@ -21,6 +21,9 @@ import org.apache.slider.core.exceptions.SliderException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.List; + /** * Application metainfo uber class */ @@ -71,6 +74,16 @@ public class Metainfo { return null; } + public List<ConfigFile> getComponentConfigFiles(String roleGroup) { + List<ConfigFile> componentConfigFiles = new ArrayList<>(); + componentConfigFiles.addAll(application.getConfigFiles()); + Component component = getApplicationComponent(roleGroup); + if (component != null) { + componentConfigFiles.addAll(component.getConfigFiles()); + } + return componentConfigFiles; + } + public void validate() throws SliderException { if (!VERSION_TWO_ONE.equals(schemaVersion) && !VERSION_TWO_ZERO.equals(schemaVersion)) { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java index a6f0e9d..8b520eb 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java +++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java @@ -67,14 +67,25 @@ public class MetainfoParser extends AbstractMetainfoParser{ digester.addBeanPropertySetter("*/commandScript/timeout"); digester.addSetNext("*/commandScript", "addCommandScript"); + digester.addObjectCreate("*/command", ComponentCommand.class); + digester.addBeanPropertySetter("*/command/exec"); + digester.addBeanPropertySetter("*/command/name"); + digester.addBeanPropertySetter("*/command/type"); + digester.addSetNext("*/command", "addCommand"); + digester.addObjectCreate("*/osSpecific", OSSpecific.class); digester.addBeanPropertySetter("*/osSpecific/osType"); - digester.addObjectCreate("*/package", OSPackage.class); - digester.addBeanPropertySetter("*/package/type"); - digester.addBeanPropertySetter("*/package/name"); - digester.addSetNext("*/package", "addOSPackage"); + digester.addObjectCreate("*/osSpecific/packages/package", OSPackage.class); + digester.addBeanPropertySetter("*/osSpecific/packages/package/type"); + digester.addBeanPropertySetter("*/osSpecific/packages/package/name"); + digester.addSetNext("*/osSpecific/packages/package", "addOSPackage"); digester.addSetNext("*/osSpecific", "addOSSpecific"); + digester.addObjectCreate("*/application/packages/package", Package.class); + digester.addBeanPropertySetter("*/application/packages/package/type"); + digester.addBeanPropertySetter("*/application/packages/package/name"); + digester.addSetNext("*/application/packages/package", "addPackage"); + digester.addObjectCreate("*/configFile", ConfigFile.class); digester.addBeanPropertySetter("*/configFile/type"); digester.addBeanPropertySetter("*/configFile/fileName"); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/test/app_packages/test_am_config/appConfig.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_am_config/appConfig.json b/slider-core/src/test/app_packages/test_am_config/appConfig.json new file mode 100644 index 0000000..9ede591 --- /dev/null +++ b/slider-core/src/test/app_packages/test_am_config/appConfig.json @@ -0,0 +1,31 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": { + }, + "global": { + "am.config.generation": "true", + "site.global.application_id": "DateLogger", + "site.global.app_version": "1.0.0", + "site.global.app_root": "${AGENT_WORK_ROOT}/app/packages/command-logger", + + "site.cl-site.logfile.location": "${AGENT_WORK_ROOT}/app/install/command-logger-app/operations.log", + "site.cl-site.datetime.format": "%A, %d. %B %Y %I:%M%p", + "site.cl-site.pattern.for.test.to.verify": "verify this pattern", + + "site.client-json.clientkey": "clientval", + "site.test-json.jsonkey": "val1", + "site.test-xml.xmlkey": "val2", + "site.test-hadoop-xml.xmlkey": "val3", + "site.test-properties.propkey": "val4", + "site.test-yaml.yamlkey": "val5", + "site.test-env.content": "test ${envkey1} {{envkey2}} content", + "site.test-env.envkey1": "envval1", + "site.test-env.envkey2": "envval2", + "site.test-template.templatekey1": "templateval1", + "site.test-template.templatekey2": "templateval2" + }, + "components": { + "COMMAND_LOGGER": { + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/test/app_packages/test_am_config/metainfo.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_am_config/metainfo.json b/slider-core/src/test/app_packages/test_am_config/metainfo.json new file mode 100644 index 0000000..0238fd3 --- /dev/null +++ b/slider-core/src/test/app_packages/test_am_config/metainfo.json @@ -0,0 +1,70 @@ +{ + "schemaVersion": "2.1", + "application": { + "name": "DATE_LOGGER", + "components": [ + { + "name": "DATE_LOGGER", + "commands": [ + { + "exec": "echo \"Time: `date +{$conf:@//site/cl-site/datetime.format}` > {$conf:@//site/cl-site/logfile.location} 2>&1 && sleep 180000" + } + ] + }, + { + "name": "DATE_LOGGER_CLIENT", + "category": "CLIENT", + "configFiles": [ + { + "type": "json", + "fileName": "client.json", + "dictionaryName": "client-json" + } + ] + } + ], + "packages": [ + { + "type": "archive", + "name": "test_am_config_generation.tgz" + } + ], + "configFiles": [ + { + "type": "properties", + "fileName": "test.properties", + "dictionaryName": "test-properties" + }, + { + "type": "template", + "fileName": "test.template", + "dictionaryName": "test-template" + }, + { + "type": "json", + "fileName": "test.json", + "dictionaryName": "test-json" + }, + { + "type": "env", + "fileName": "testenv", + "dictionaryName": "test-env" + }, + { + "type": "hadoop-xml", + "fileName": "test-hadoop.xml", + "dictionaryName": "test-hadoop-xml" + }, + { + "type": "xml", + "fileName": "test.xml", + "dictionaryName": "test-xml" + }, + { + "type": "yaml", + "fileName": "test.yaml", + "dictionaryName": "test-yaml" + } + ] + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/test/app_packages/test_am_config/resources.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_am_config/resources.json b/slider-core/src/test/app_packages/test_am_config/resources.json new file mode 100644 index 0000000..72c2b7b --- /dev/null +++ b/slider-core/src/test/app_packages/test_am_config/resources.json @@ -0,0 +1,17 @@ +{ + "schema" : "http://example.org/specification/v2.0.0";, + "metadata" : { + }, + "global" : { + }, + "components": { + "slider-appmaster": { + "yarn.memory": "256" + }, + "DATE_LOGGER": { + "yarn.role.priority": "1", + "yarn.component.instances": "1", + "yarn.memory": "128" + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/test/app_packages/test_am_config/resources/test.template ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_am_config/resources/test.template b/slider-core/src/test/app_packages/test_am_config/resources/test.template new file mode 100644 index 0000000..1aff6bd --- /dev/null +++ b/slider-core/src/test/app_packages/test_am_config/resources/test.template @@ -0,0 +1 @@ +test ${templatekey1} {{templatekey2}} content http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/test/app_packages/test_am_config/test_archive/testfile ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_am_config/test_archive/testfile b/slider-core/src/test/app_packages/test_am_config/test_archive/testfile new file mode 100644 index 0000000..8cd6f83 --- /dev/null +++ b/slider-core/src/test/app_packages/test_am_config/test_archive/testfile @@ -0,0 +1 @@ +test archive contents http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/test/app_packages/test_command_log/resources_unique_names.json ---------------------------------------------------------------------- diff --git a/slider-core/src/test/app_packages/test_command_log/resources_unique_names.json b/slider-core/src/test/app_packages/test_command_log/resources_unique_names.json new file mode 100644 index 0000000..46b0629 --- /dev/null +++ b/slider-core/src/test/app_packages/test_command_log/resources_unique_names.json @@ -0,0 +1,18 @@ +{ + "schema": "http://example.org/specification/v2.0.0";, + "metadata": { + }, + "global": { + }, + "components": { + "COMMAND_LOGGER": { + "component.unique.names": "true", + "yarn.memory": "128", + "yarn.role.priority": "1", + "yarn.component.instances": "2" + }, + "slider-appmaster": { + "yarn.memory": "256" + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java b/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java new file mode 100644 index 0000000..63d5961 --- /dev/null +++ b/slider-core/src/test/java/org/apache/slider/core/registry/docstore/TestPublishedConfigurationOutputter.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.slider.core.registry.docstore; + +import com.google.common.base.Charsets; +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.fs.Path; +import org.apache.slider.common.tools.SliderFileSystem; +import org.codehaus.jackson.map.ObjectMapper; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.powermock.api.easymock.PowerMock; +import org.yaml.snakeyaml.Yaml; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.expect; +import static org.mockito.Matchers.anyString; +import static org.powermock.api.easymock.PowerMock.createNiceMock; + +public class TestPublishedConfigurationOutputter { + private static HashMap<String, String> config = new HashMap<>(); + + @Rule + public TemporaryFolder tmpDir = new TemporaryFolder(); + + @Before + public void setup() { + config.put("key1", "val1"); + } + + @Test + public void testJson() throws IOException { + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter(ConfigFormat.JSON, + new PublishedConfiguration("description", + config.entrySet())); + + String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)", + ""); + assert "{\"key1\":\"val1\"}".equals(output); + + File file = tmpDir.newFile(); + configurationOutputter.save(file); + + ObjectMapper mapper = new ObjectMapper(); + Map<String, String> read = mapper.readValue(file, Map.class); + assert 1 == read.size(); + assert "val1".equals(read.get("key1")); + } + + @Test + public void testXml() throws IOException { + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter(ConfigFormat.XML, + new PublishedConfiguration("description", + config.entrySet())); + + String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)", + ""); + assert output.contains( + "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>"); + + File file = tmpDir.newFile(); + configurationOutputter.save(file); + + assert FileUtils.readFileToString(file, Charsets.UTF_8) + .replaceAll("( |\\r|\\n)", "") + .contains( + "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>"); + } + + @Test + public void testHadoopXml() throws IOException { + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter(ConfigFormat.HADOOP_XML, + new PublishedConfiguration("description", + config.entrySet())); + + String output = configurationOutputter.asString().replaceAll("( |\\r|\\n)", + ""); + assert output.contains("<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>"); + + File file = tmpDir.newFile(); + configurationOutputter.save(file); + + assert FileUtils.readFileToString(file, Charsets.UTF_8) + .replaceAll("( |\\r|\\n)", "") + .contains( "<configuration><property><name>key1</name><value>val1</value><source/></property></configuration>"); + } + + @Test + public void testProperties() throws IOException { + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter(ConfigFormat.PROPERTIES, + new PublishedConfiguration("description", + config.entrySet())); + + String output = configurationOutputter.asString(); + assert output.contains("key1=val1"); + + File file = tmpDir.newFile(); + configurationOutputter.save(file); + + Properties properties = new Properties(); + FileInputStream fis = null; + try { + fis = new FileInputStream(file); + properties.load(fis); + } finally { + if (fis != null) { + fis.close(); + } + } + assert 1 == properties.size(); + assert "val1".equals(properties.getProperty("key1")); + } + + @Test + public void testYaml() throws IOException { + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter(ConfigFormat.YAML, + new PublishedConfiguration("description", + config.entrySet())); + + String output = configurationOutputter.asString().replaceAll("(\\r|\\n)", + ""); + assert "key1: val1".equals(output); + + File file = tmpDir.newFile(); + configurationOutputter.save(file); + + Yaml yaml = new Yaml(); + FileInputStream fis = null; + Map<String, String> read; + try { + fis = new FileInputStream(file); + read = (Map<String, String>) yaml.load(fis); + } finally { + if (fis != null) { + fis.close(); + } + } + assert 1 == read.size(); + assert "val1".equals(read.get("key1")); + } + + @Test + public void testEnv() throws IOException { + HashMap<String, String> envConfig = new HashMap<>(config); + envConfig.put("content", "content {{key1}} "); + + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter(ConfigFormat.ENV, + new PublishedConfiguration("description", + envConfig.entrySet())); + + String output = configurationOutputter.asString(); + assert "content val1 ".equals(output); + + File file = tmpDir.newFile(); + configurationOutputter.save(file); + + assert "content val1 ".equals(FileUtils.readFileToString(file, + Charsets.UTF_8)); + } + + @Test + public void testTemplate1() throws IOException { + HashMap<String, String> templateConfig = new HashMap<>(config); + templateConfig.put(ConfigUtils.TEMPLATE_FILE, "templateFileName"); + + SliderFileSystem fileSystem = createNiceMock(SliderFileSystem.class); + expect(fileSystem.buildResourcePath(anyString())).andReturn(new Path("path")).anyTimes(); + expect(fileSystem.isFile(anyObject(Path.class))).andReturn(true).anyTimes(); + expect(fileSystem.cat(anyObject(Path.class))).andReturn("content {{key1}}\n more ${key1} content").anyTimes(); + + PowerMock.replay(fileSystem); + + ConfigUtils.prepConfigForTemplateOutputter(ConfigFormat.TEMPLATE, + templateConfig, fileSystem, "clusterName", null); + PublishedConfigurationOutputter configurationOutputter = + PublishedConfigurationOutputter.createOutputter(ConfigFormat.TEMPLATE, + new PublishedConfiguration("description", + templateConfig.entrySet())); + + String output = configurationOutputter.asString(); + assert "content val1\n more val1 content".equals(output); + + File file = tmpDir.newFile(); + configurationOutputter.save(file); + + PowerMock.verify(fileSystem); + + assert "content val1\n more val1 content".equals( + FileUtils.readFileToString(file, Charsets.UTF_8)); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java index b919fcf..b959e2f 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java @@ -239,7 +239,11 @@ public class TestAgentClientProvider2 extends SliderTestUtils { public void testSliderClientForInstallFailures() throws Exception { describe(" IGNORE ANY STACK TRACES BELOW "); - SliderClient client = new SliderClient(); + SliderClient client = PowerMock.createPartialMock(SliderClient.class, + "getRegistryOperations"); + expect(client.getRegistryOperations()).andReturn(null).anyTimes(); + PowerMock.replay(SliderClient.class); + client.bindArgs(new Configuration(), "client", "--dest", "a_random_path/none", "--package", "a_random_pkg.zip"); ActionClientArgs args = new ActionClientArgs(); @@ -287,5 +291,7 @@ public class TestAgentClientProvider2 extends SliderTestUtils { assertExceptionDetails(e, SliderExitCodes.EXIT_BAD_CONFIGURATION, AgentClientProvider.E_COULD_NOT_READ_METAINFO); } + + PowerMock.verify(SliderClient.class); } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java index 9fbb3d0..0d56ff7 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java @@ -88,11 +88,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.ByteArrayInputStream; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -285,6 +287,7 @@ public class TestAgentProviderService { ProviderRole role = new ProviderRole("HBASE_MASTER", 1); SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class); ContainerLauncher launcher = createNiceMock(ContainerLauncher.class); + expect(launcher.getEnv()).andReturn(new MapOperations()).anyTimes(); Path generatedConfPath = new Path(".", "test"); MapOperations resourceComponent = new MapOperations(); MapOperations appComponent = new MapOperations(); @@ -347,6 +350,9 @@ public class TestAgentProviderService { anyString(), anyString() ); + + doReturn(Collections.emptyMap()).when(mockAps).getRoleClusterNodeMapping(); + expect(access.isApplicationLive()).andReturn(true).anyTimes(); ClusterDescription desc = new ClusterDescription(); desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181"); @@ -357,16 +363,15 @@ public class TestAgentProviderService { expect(access.getRoleClusterNodeMapping()).andReturn(cnMap).anyTimes(); - AggregateConf aggConf = new AggregateConf(); - ConfTreeOperations treeOps = aggConf.getAppConfOperations(); + ConfTreeOperations treeOps = instanceDefinition.getAppConfOperations(); treeOps.getOrAddComponent("HBASE_MASTER").put(AgentKeys.WAIT_HEARTBEAT, "0"); treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE"); treeOps.set("site.fs.defaultFS", "hdfs://HOST1:8020/"); treeOps.set("internal.data.dir.path", "hdfs://HOST1:8020/database"); - expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf); + expect(access.getInstanceDefinitionSnapshot()).andReturn(instanceDefinition); expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes(); expect(access.getAppConfSnapshot()).andReturn(treeOps).anyTimes(); - replay(access, ctx, container, sliderFileSystem, mockFs); + replay(access, ctx, container, sliderFileSystem, mockFs, launcher); try { mockAps.buildContainerLaunchContext(launcher, @@ -897,6 +902,8 @@ public class TestAgentProviderService { doReturn(access).when(mockAps).getAmState(); PublishedExportsSet pubExpSet = new PublishedExportsSet(); expect(access.getPublishedExportsSet()).andReturn(pubExpSet).anyTimes(); + expect(access.getAppConfSnapshot()).andReturn(new ConfTreeOperations( + new ConfTree())).anyTimes(); replay(access); Map<String, String> ports = new HashMap<String, String>(); @@ -1011,6 +1018,8 @@ public class TestAgentProviderService { doReturn(access).when(mockAps).getAmState(); PublishedExportsSet pubExpSet = new PublishedExportsSet(); expect(access.getPublishedExportsSet()).andReturn(pubExpSet).anyTimes(); + expect(access.getAppConfSnapshot()).andReturn(new ConfTreeOperations( + new ConfTree())).anyTimes(); replay(access); mockAps.publishConfigAndExportGroups(hb, componentStatus, "HBASE_MASTER"); @@ -1225,6 +1234,8 @@ public class TestAgentProviderService { SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class); ContainerLauncher launcher = createNiceMock(ContainerLauncher.class); ContainerLauncher launcher2 = createNiceMock(ContainerLauncher.class); + expect(launcher.getEnv()).andReturn(new MapOperations()).anyTimes(); + expect(launcher2.getEnv()).andReturn(new MapOperations()).anyTimes(); Path generatedConfPath = new Path(".", "test"); MapOperations resourceComponent = new MapOperations(); MapOperations appComponent = new MapOperations(); @@ -1291,18 +1302,28 @@ public class TestAgentProviderService { desc.setInfo(OptionKeys.APPLICATION_NAME, "HBASE"); expect(access.getClusterStatus()).andReturn(desc).anyTimes(); - AggregateConf aggConf = new AggregateConf(); - ConfTreeOperations treeOps = aggConf.getAppConfOperations(); + ConfTreeOperations treeOps = instanceDefinition.getAppConfOperations(); treeOps.getOrAddComponent("HBASE_MASTER").put(AgentKeys.WAIT_HEARTBEAT, "0"); - treeOps.getOrAddComponent("HBASE_REGIONSERVER").put(AgentKeys.WAIT_HEARTBEAT, "0"); + treeOps.getOrAddComponent("HBASE_REGIONSERVER").put( + AgentKeys.WAIT_HEARTBEAT, "0"); treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE"); treeOps.set("site.fs.defaultFS", "hdfs://HOST1:8020/"); treeOps.set("internal.data.dir.path", "hdfs://HOST1:8020/database"); - expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf).anyTimes(); + expect(access.getInstanceDefinitionSnapshot()).andReturn(instanceDefinition).anyTimes(); expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes(); expect(access.getAppConfSnapshot()).andReturn(treeOps).anyTimes(); - doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection()); - replay(access, ctx, container, sliderFileSystem, mockFs); + doNothing().when(mockAps).publishApplicationInstanceData(anyString(), + anyString(), anyCollection()); + doNothing().when(mockAps).localizeConfigFiles( + (ContainerLauncher)Matchers.anyObject(), + anyString(), + anyString(), + (Metainfo)Matchers.anyObject(), + anyMap(), + (MapOperations)Matchers.anyObject(), + (SliderFileSystem)Matchers.anyObject()); + doReturn(Collections.emptyMap()).when(mockAps).getRoleClusterNodeMapping(); + replay(access, ctx, container, sliderFileSystem, mockFs, launcher, launcher2); // build two containers try { @@ -1849,12 +1870,12 @@ public class TestAgentProviderService { AgentProviderService aps = createAgentProviderService(new Configuration()); Map<String, Map<String, String>> allConfigs = new HashMap<String, Map<String, String>>(); Map<String, String> cfg1 = new HashMap<String, String>(); - cfg1.put("a1", "${@//site/cfg-2/A1}"); + cfg1.put("a1", "0${@//site/cfg-2/A1}"); cfg1.put("b1", "22"); cfg1.put("c1", "33"); cfg1.put("d1", "${@//site/cfg1/c1}AA"); Map<String, String> cfg2 = new HashMap<String, String>(); - cfg2.put("A1", "11"); + cfg2.put("A1", "11${@//site/cfg1/b1}"); cfg2.put("B1", "${@//site/cfg-2/A1},${@//site/cfg-2/A1},AA,${@//site/cfg1/c1}"); cfg2.put("C1", "DD${@//site/cfg1/c1}"); cfg2.put("D1", "${14}"); @@ -1862,15 +1883,30 @@ public class TestAgentProviderService { allConfigs.put("cfg1", cfg1); allConfigs.put("cfg-2", cfg2); aps.dereferenceAllConfigs(allConfigs); - Assert.assertEquals("11", cfg1.get("a1")); + Assert.assertEquals("01122", cfg1.get("a1")); Assert.assertEquals("22", cfg1.get("b1")); Assert.assertEquals("33", cfg1.get("c1")); Assert.assertEquals("33AA", cfg1.get("d1")); - Assert.assertEquals("11", cfg2.get("A1")); - Assert.assertEquals("11,11,AA,33", cfg2.get("B1")); + Assert.assertEquals("1122", cfg2.get("A1")); + Assert.assertEquals("1122,1122,AA,33", cfg2.get("B1")); Assert.assertEquals("DD33", cfg2.get("C1")); Assert.assertEquals("${14}", cfg2.get("D1")); } + @Test + public void testDereferenceAllConfigLoop() throws IOException { + AgentProviderService aps = createAgentProviderService(new Configuration()); + Map<String, Map<String, String>> allConfigs = new HashMap<String, Map<String, String>>(); + Map<String, String> cfg1 = new HashMap<String, String>(); + cfg1.put("a1", "0${@//site/cfg-2/A1}"); + Map<String, String> cfg2 = new HashMap<String, String>(); + cfg2.put("A1", "11${@//site/cfg1/a1}"); + + allConfigs.put("cfg1", cfg1); + allConfigs.put("cfg-2", cfg2); + aps.dereferenceAllConfigs(allConfigs); + Assert.assertEquals("0${@//site/cfg-2/A1}", cfg1.get("a1")); + Assert.assertEquals("11${@//site/cfg1/a1}", cfg2.get("A1")); + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java ---------------------------------------------------------------------- diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java index 30283d1..ba1912a 100644 --- a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java +++ b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java @@ -50,7 +50,7 @@ public class TestMetainfoParser { Assert.assertNotNull(metainfo.getApplication()); Application application = metainfo.getApplication(); assert "STORM".equals(application.getName()); - assert 5 == application.getComponents().size(); + assert 6 == application.getComponents().size(); OSPackage pkg = application.getOSSpecifics().get(0).getPackages().get(0); assert "tarball".equals(pkg.getType()); assert "files/apache-storm-0.9.1.2.1.1.0-237.tar.gz".equals(pkg.getName()); @@ -63,9 +63,20 @@ public class TestMetainfoParser { if (comp != null && comp.getName().equals("SUPERVISOR")) { Assert.assertEquals(1, comp.getComponentExports().size()); } + if (comp != null && comp.getName().equals("ANOTHER_COMPONENT")) { + assert 2 == comp.getCommands().size(); + assert "start command".equals(comp.getCommands().get(0).getExec()); + assert "START".equals(comp.getCommands().get(0).getName()); + assert "stop command".equals(comp.getCommands().get(1).getExec()); + assert "STOP".equals(comp.getCommands().get(1).getName()); + } } assert found; Assert.assertEquals(0, application.getConfigFiles().size()); + assert 1 == application.getPackages().size(); + Package p = application.getPackages().get(0); + assert "tarball".equals(p.getType()); + assert "test-tarball-name.tgz".equals(p.getName()); } @Test http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml ---------------------------------------------------------------------- diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml b/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml index d9004ad..fbe9299 100644 --- a/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml +++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/application/metadata/metainfo.xml @@ -142,6 +142,20 @@ <timeout>600</timeout> </commandScript> </component> + + <component> + <name>ANOTHER_COMPONENT</name> + <category>MASTER</category> + <commands> + <command> + <exec>start command</exec> + </command> + <command> + <exec>stop command</exec> + <name>STOP</name> + </command> + </commands> + </component> </components> <osSpecifics> @@ -155,5 +169,12 @@ </packages> </osSpecific> </osSpecifics> + + <packages> + <package> + <type>tarball</type> + <name>test-tarball-name.tgz</name> + </package> + </packages> </application> </metainfo> http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy index 5de2b8e..c0aa06a 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy @@ -38,4 +38,9 @@ interface ResourcePaths { String SLEEP_META = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/metainfo.json" String SLEEP_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig.json" + String AM_CONFIG_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_am_config/resources.json" + String AM_CONFIG_META = "$SLIDER_CORE_APP_PACKAGES/test_am_config/metainfo.json" + String AM_CONFIG_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_am_config/appConfig.json" + + String UNIQUE_COMPONENT_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_command_log/resources_unique_names.json" } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/AMConfigPublishingIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/AMConfigPublishingIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/AMConfigPublishingIT.groovy new file mode 100644 index 0000000..e84ada6 --- /dev/null +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/AMConfigPublishingIT.groovy @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.funtest.misc + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.slider.common.SliderExitCodes +import org.apache.slider.common.params.Arguments +import org.apache.slider.common.params.SliderActions +import org.apache.slider.common.tools.SliderUtils +import org.apache.slider.funtest.ResourcePaths +import org.apache.slider.funtest.framework.AgentCommandTestBase +import org.apache.slider.funtest.framework.FuntestProperties +import org.apache.slider.funtest.framework.SliderShell +import org.junit.After +import org.junit.Before +import org.junit.Test + +@CompileStatic +@Slf4j +public class AMConfigPublishingIT extends AgentCommandTestBase +implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { + + private static String DATE_LOGGER = "DATE_LOGGER" + private static String APP_NAME = "am-config-publishing" + private static String APP_METAINFO = ResourcePaths.AM_CONFIG_META + private static String APP_RESOURCE = ResourcePaths.AM_CONFIG_RESOURCES + private static String APP_TEMPLATE = ResourcePaths.AM_CONFIG_APPCONFIG + private static String CLIENT_CONFIG = "../slider-core/src/test/app_packages/test_am_config/clientInstallConfig-default.json" + private static String RESOURCE_DIR = "../slider-core/src/test/app_packages/test_am_config/resources" + private static String TGZ_SOURCE = "../slider-core/src/test/app_packages/test_am_config/test_archive" + private static String TGZ_FILE = "test_am_config_generation.tgz" + private static String TGZ_DIR = "target/package-tmp/" + + private static String CLIENT_INSTALL_DIR = "target/am-config-client" + + private HashMap<String, String> files = + ["client.json": """{ "clientkey" : "clientval"}""", + "test.json": """{ "jsonkey" : "val1"}""", + "test.xml": "<configuration><property><name>xmlkey</name><value>val2</value><source/></property></configuration>", + "test-hadoop.xml": "<configuration><property><name>xmlkey</name><value>val3</value><source/></property></configuration>", + "test.properties": "propkey=val4", + "test.yaml": "yamlkey: val5", + "test.template": "test templateval1 templateval2 content", + "testenv": "test envval1 envval2 content", + "testfile": "test archive contents" + ] + + @Before + public void prepareCluster() { + setupCluster(APP_NAME) + } + + @Before + public void setupApplicationPackage() { + File tgzFile = new File(TGZ_DIR + TGZ_FILE); + SliderUtils.tarGzipFolder(new File(TGZ_SOURCE), tgzFile, null); + try { + tgzFile = tgzFile.canonicalFile + SliderShell shell = slider(EXIT_SUCCESS, + [ + ACTION_RESOURCE, + ARG_INSTALL, ARG_RESOURCE, tgzFile.absolutePath, + ARG_OVERWRITE + ]) + logShell(shell) + shell = slider(EXIT_SUCCESS, + [ + ACTION_RESOURCE, + ARG_INSTALL, ARG_RESOURCE, RESOURCE_DIR, + ARG_DESTDIR, APP_NAME, + ARG_OVERWRITE + ]) + logShell(shell) + log.info "Resources uploaded at home directory .slider/resources" + } catch (Exception e) { + setup_failed = true + throw e; + } + File dir = new File(CLIENT_INSTALL_DIR) + if (!dir.exists()) { + dir.mkdir() + } + } + + + @After + public void destroyCluster() { + cleanup(APP_NAME) + } + + @Test + public void testCreate() throws Throwable { + assumeAgentTestsEnabled() + + describe APP_NAME + + def path = buildClusterPath(APP_NAME) + assert !clusterFS.exists(path) + + slider(EXIT_SUCCESS, + [ + ACTION_CREATE, APP_NAME, + ARG_TEMPLATE, APP_TEMPLATE, ARG_RESOURCES, APP_RESOURCE, + ARG_METAINFO, APP_METAINFO + ]) + ensureApplicationIsUp(APP_NAME) + + expectLiveContainerCountReached(APP_NAME, DATE_LOGGER, 1, + CONTAINER_LAUNCH_TIMEOUT) + status(0, APP_NAME) + + SliderShell shell = slider(EXIT_SUCCESS, + [ + ACTION_CLIENT, ARG_INSTALL, + ARG_DEST, CLIENT_INSTALL_DIR, + ARG_NAME, APP_NAME + ]) + logShell(shell) + + for (Map.Entry<String, String> entry : files.entrySet()) { + String name = entry.getKey(); + File file = new File(CLIENT_INSTALL_DIR + "/" + name) + assert file.exists() + String contents = file.text.replaceAll("(\\r|\\n)", "") + assert contents.contains(entry.getValue()), "$name didn't contain value" + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/424b2a40/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/UniqueComponentNamesIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/UniqueComponentNamesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/UniqueComponentNamesIT.groovy new file mode 100644 index 0000000..110de22 --- /dev/null +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/UniqueComponentNamesIT.groovy @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.funtest.misc + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import org.apache.slider.api.ClusterDescription +import org.apache.slider.common.SliderExitCodes +import org.apache.slider.common.SliderKeys +import org.apache.slider.common.params.Arguments +import org.apache.slider.common.params.SliderActions +import org.apache.slider.funtest.ResourcePaths +import org.apache.slider.funtest.framework.AgentCommandTestBase +import org.apache.slider.funtest.framework.FuntestProperties +import org.apache.slider.funtest.framework.SliderShell +import org.junit.After +import org.junit.Before +import org.junit.Test + +@CompileStatic +@Slf4j +public class UniqueComponentNamesIT extends AgentCommandTestBase +implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { + + private static String COMMAND_LOGGER = "COMMAND_LOGGER" + private static String COMMAND_LOGGER1 = COMMAND_LOGGER + "1" + private static String COMMAND_LOGGER2 = COMMAND_LOGGER + "2" + private static String COMMAND_LOGGER3 = COMMAND_LOGGER + "3" + private static String APPLICATION_NAME = "unique-component-names" + private static String APP_RESOURCE = ResourcePaths.UNIQUE_COMPONENT_RESOURCES + + @Before + public void prepareCluster() { + setupCluster(APPLICATION_NAME) + } + + @After + public void destroyCluster() { + cleanup(APPLICATION_NAME) + } + + @Test + public void testCreateFlex() throws Throwable { + assumeAgentTestsEnabled() + + describe APPLICATION_NAME + + def path = buildClusterPath(APPLICATION_NAME) + assert !clusterFS.exists(path) + + File launchReportFile = createTempJsonFile(); + SliderShell shell = createTemplatedSliderApplication(APPLICATION_NAME, + APP_TEMPLATE, + APP_RESOURCE, + [], + launchReportFile) + logShell(shell) + + ensureYarnApplicationIsUp(launchReportFile) + ensureApplicationIsUp(APPLICATION_NAME) + + ClusterDescription cd = execStatus(APPLICATION_NAME) + + assert 3 == cd.statistics.size() + assert cd.statistics.keySet().containsAll([SliderKeys.COMPONENT_AM, COMMAND_LOGGER1, COMMAND_LOGGER2]) + + slider(EXIT_SUCCESS, + [ + ACTION_FLEX, + APPLICATION_NAME, + ARG_COMPONENT, + COMMAND_LOGGER, + "3" + ]) + + sleep(1000 * 10) + + status(0, APPLICATION_NAME) + expectLiveContainerCountReached(APPLICATION_NAME, COMMAND_LOGGER3, 1, + CONTAINER_LAUNCH_TIMEOUT) + expectLiveContainerCountReached(APPLICATION_NAME, COMMAND_LOGGER2, 1, + CONTAINER_LAUNCH_TIMEOUT) + expectLiveContainerCountReached(APPLICATION_NAME, COMMAND_LOGGER1, 1, + CONTAINER_LAUNCH_TIMEOUT) + + slider(EXIT_SUCCESS, + [ + ACTION_FLEX, + APPLICATION_NAME, + ARG_COMPONENT, + COMMAND_LOGGER, + "1" + ]) + + sleep(1000 * 10) + + status(0, APPLICATION_NAME) + expectLiveContainerCountReached(APPLICATION_NAME, COMMAND_LOGGER3, 0, + CONTAINER_LAUNCH_TIMEOUT) + expectLiveContainerCountReached(APPLICATION_NAME, COMMAND_LOGGER2, 0, + CONTAINER_LAUNCH_TIMEOUT) + expectLiveContainerCountReached(APPLICATION_NAME, COMMAND_LOGGER1, 1, + CONTAINER_LAUNCH_TIMEOUT) + } + +}
