AMBARI-7239. Ambari items in /tmp (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/adb87a44 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/adb87a44 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/adb87a44 Branch: refs/heads/branch-alerts-dev Commit: adb87a44792eaa588ace009ac92e68926ba0cc37 Parents: 3f9d5a5 Author: Andrew Onishuk <aonis...@hortonworks.com> Authored: Wed Sep 10 19:25:27 2014 +0300 Committer: Andrew Onishuk <aonis...@hortonworks.com> Committed: Wed Sep 10 19:25:27 2014 +0300 ---------------------------------------------------------------------- ambari-server/conf/unix/ambari.properties | 1 + .../server/api/services/ComponentService.java | 5 +++- .../api/services/HostComponentService.java | 5 +++- .../server/configuration/Configuration.java | 4 +++ .../internal/ClientConfigResourceProvider.java | 5 ++-- ambari-server/src/main/python/bootstrap.py | 24 ++++++++++++++-- .../ClientConfigResourceProviderTest.java | 16 +++++++---- ambari-server/src/test/python/TestBootstrap.py | 29 ++++++++++++++++---- 8 files changed, 71 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/adb87a44/ambari-server/conf/unix/ambari.properties ---------------------------------------------------------------------- diff --git a/ambari-server/conf/unix/ambari.properties b/ambari-server/conf/unix/ambari.properties index b77ae32..41cada7 100644 --- a/ambari-server/conf/unix/ambari.properties +++ b/ambari-server/conf/unix/ambari.properties @@ -31,6 +31,7 @@ bootstrap.script=/usr/lib/python2.6/site-packages/ambari_server/bootstrap.py bootstrap.setup_agent.script=/usr/lib/python2.6/site-packages/ambari_server/setupAgent.py recommendations.dir=/var/run/ambari-server/stack-recommendations stackadvisor.script=/var/lib/ambari-server/resources/scripts/stack_advisor.py +server.tmp.dir=/var/lib/ambari-server/tmp api.authenticate=true server.connection.max.idle.millis=900000 http://git-wip-us.apache.org/repos/asf/ambari/blob/adb87a44/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java index 16a8ffa..5510697 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java @@ -20,6 +20,7 @@ package org.apache.ambari.server.api.services; import com.google.inject.Inject; import org.apache.ambari.server.api.resources.ResourceInstance; +import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.spi.ClusterController; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.utilities.ClusterControllerHelper; @@ -231,7 +232,9 @@ public class ComponentService extends BaseService { } Response.ResponseBuilder rb = Response.status(Response.Status.OK); - File file = new File("/tmp/ambari-server/"+componentName+"-configs.tar.gz"); + Configuration configs = new Configuration(); + String tmpDir = configs.getProperty(Configuration.SERVER_TMP_DIR_KEY); + File file = new File(tmpDir+File.separator+componentName+"-configs.tar.gz"); InputStream resultInputStream = null; try { resultInputStream = new FileInputStream(file); http://git-wip-us.apache.org/repos/asf/ambari/blob/adb87a44/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostComponentService.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostComponentService.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostComponentService.java index 35c7826..4990ad7 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostComponentService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostComponentService.java @@ -32,6 +32,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; import org.apache.ambari.server.api.resources.ResourceInstance; +import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.spi.Resource; /** @@ -270,7 +271,9 @@ public class HostComponentService extends BaseService { } Response.ResponseBuilder rb = Response.status(Response.Status.OK); - File file = new File("/tmp/ambari-server/"+hostComponentName+"-configs.tar.gz"); + Configuration configs = new Configuration(); + String tmpDir = configs.getProperty(Configuration.SERVER_TMP_DIR_KEY); + File file = new File(tmpDir+File.separator+hostComponentName+"-configs.tar.gz"); InputStream resultInputStream = null; try { resultInputStream = new FileInputStream(file); http://git-wip-us.apache.org/repos/asf/ambari/blob/adb87a44/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index a21f98c..a3a07b0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -231,6 +231,8 @@ public class Configuration { public static final String DEFAULT_SCHEDULER_MAX_CONNECTIONS = "5"; public static final String DEFAULT_EXECUTION_SCHEDULER_MISFIRE_TOLERATION = "480"; public static final String DEFAULT_SCHEDULER_START_DELAY_SECONDS = "120"; + public static final String SERVER_TMP_DIR_KEY = "server.tmp.dir"; + public static final String SERVER_TMP_DIR_DEFAULT = "/var/lib/ambari-server/tmp"; /** * This key defines whether stages of parallel requests are executed in * parallel or sequentally. Only stages from different requests @@ -372,6 +374,8 @@ public class Configuration { JAVA_HOME_KEY)); configsMap.put(PARALLEL_STAGE_EXECUTION_KEY, properties.getProperty( PARALLEL_STAGE_EXECUTION_KEY, PARALLEL_STAGE_EXECUTION_DEFAULT)); + configsMap.put(SERVER_TMP_DIR_KEY, properties.getProperty( + SERVER_TMP_DIR_KEY, SERVER_TMP_DIR_DEFAULT)); File passFile = new File(configsMap.get(SRVR_KSTR_DIR_KEY) + File.separator + configsMap.get(SRVR_CRT_PASS_FILE_KEY)); http://git-wip-us.apache.org/repos/asf/ambari/blob/adb87a44/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java index cc97eba..366b5e1 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java @@ -55,7 +55,6 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv protected static final String COMPONENT_COMPONENT_NAME_PROPERTY_ID = "ServiceComponentInfo/component_name"; protected static final String HOST_COMPONENT_HOST_NAME_PROPERTY_ID = PropertyHelper.getPropertyId("HostRoles", "host_name"); - protected static final String TMP_PATH = "/tmp/ambari-server"; private final Gson gson; @@ -121,6 +120,8 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv throw new SystemException("Failed to get components ", e); } + Configuration configs = new Configuration(); + String TMP_PATH = configs.getProperty(Configuration.SERVER_TMP_DIR_KEY); AmbariManagementController managementController = getManagementController(); ConfigHelper configHelper = managementController.getConfigHelper(); Cluster cluster = null; @@ -289,7 +290,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv File tmpDirectory = new File(jsonFileName.getParent()); if (!tmpDirectory.exists()) { try { - tmpDirectory.mkdir(); + tmpDirectory.mkdirs(); tmpDirectory.setWritable(true, true); tmpDirectory.setReadable(true, true); } catch (SecurityException se) { http://git-wip-us.apache.org/repos/asf/ambari/blob/adb87a44/ambari-server/src/main/python/bootstrap.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/python/bootstrap.py b/ambari-server/src/main/python/bootstrap.py index b981572..5ebfbf2 100755 --- a/ambari-server/src/main/python/bootstrap.py +++ b/ambari-server/src/main/python/bootstrap.py @@ -39,7 +39,8 @@ MAX_PARALLEL_BOOTSTRAPS = 20 # How many seconds to wait between polling parallel bootstraps POLL_INTERVAL_SEC = 1 DEBUG = False -PYTHON_ENV="env PYTHONPATH=$PYTHONPATH:/tmp " +DEFAULT_AGENT_TEMP_FOLDER = "/var/lib/ambari-agent/data/tmp" +PYTHON_ENV="env PYTHONPATH=$PYTHONPATH:{0} ".format(DEFAULT_AGENT_TEMP_FOLDER) class HostLog: @@ -142,7 +143,7 @@ class SSH: class Bootstrap(threading.Thread): """ Bootstrap the agent on a separate host""" - TEMP_FOLDER = "/tmp" + TEMP_FOLDER = DEFAULT_AGENT_TEMP_FOLDER OS_CHECK_SCRIPT_FILENAME = "os_check_type.py" AMBARI_REPO_FILENAME = "ambari" SETUP_SCRIPT_FILENAME = "setupAgent.py" @@ -235,6 +236,22 @@ class Bootstrap(threading.Thread): return password_file is not None and password_file != 'null' + def createTargetDir(self): + # Creating target dir + self.host_log.write("==========================\n") + self.host_log.write("Creating target directory...") + params = self.shared_state + user = params.user + + command = "[ -d {0} ] || sudo mkdir -p {0} ; sudo chown {1} {0}".format(self.TEMP_FOLDER,params.user) + + ssh = SSH(params.user, params.sshkey_file, self.host, command, + params.bootdir, self.host_log) + retcode = ssh.run() + self.host_log.write("\n") + return retcode + + def copyOsCheckScript(self): # Copying the os check script file fileToCopy = self.getOsCheckScript() @@ -489,7 +506,8 @@ class Bootstrap(threading.Thread): """ Copy files and run commands on remote host """ self.status["start_time"] = time.time() # Population of action queue - action_queue = [self.copyCommonFunctions, + action_queue = [self.createTargetDir, + self.copyCommonFunctions, self.copyOsCheckScript, self.runOsCheckScript, self.checkSudoPackage http://git-wip-us.apache.org/repos/asf/ambari/blob/adb87a44/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java index e7ecb88..214bf10 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProviderTest.java @@ -26,6 +26,7 @@ import org.apache.ambari.server.controller.utilities.PredicateBuilder; import org.apache.ambari.server.controller.utilities.PropertyHelper; import org.apache.ambari.server.state.*; import org.apache.ambari.server.state.PropertyInfo; +import org.easymock.EasyMock; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -109,8 +110,8 @@ public class ClientConfigResourceProviderTest { // create the request Request request = PropertyHelper.getUpdateRequest(properties, null); - Predicate predicate = new PredicateBuilder().property(ClientConfigResourceProvider.COMPONENT_CLUSTER_NAME_PROPERTY_ID).equals("c1"). - toPredicate(); + Predicate predicate = new PredicateBuilder().property( + ClientConfigResourceProvider.COMPONENT_CLUSTER_NAME_PROPERTY_ID).equals("c1").toPredicate(); try { provider.updateResources(request, predicate); @@ -143,13 +144,13 @@ public class ClientConfigResourceProviderTest { ServiceComponentHost serviceComponentHost = createNiceMock(ServiceComponentHost.class); ServiceOsSpecific serviceOsSpecific = createNiceMock(ServiceOsSpecific.class); ConfigHelper configHelper = createNiceMock(ConfigHelper.class); + Configuration configuration = PowerMock.createStrictMockAndExpectNew(Configuration.class); File mockFile = PowerMock.createNiceMock(File.class); Runtime runtime = createMock(Runtime.class); Process process = createNiceMock(Process.class); Collection<Config> clusterConfigs = new HashSet<Config>(); - //Config clusterConfig = new ConfigImpl("config"); clusterConfigs.add(clusterConfig); Map<String, Map<String, String>> allConfigTags = new HashMap<String, Map<String, String>>(); Map<String, Map<String, String>> properties = new HashMap<String, Map<String, String>>(); @@ -216,6 +217,7 @@ public class ClientConfigResourceProviderTest { expect(configHelper.getEffectiveConfigProperties(cluster, configTags)).andReturn(properties); expect(clusterConfig.getType()).andReturn(Configuration.HIVE_CONFIG_TAG).anyTimes(); expect(configHelper.getEffectiveConfigAttributes(cluster, configTags)).andReturn(attributes); + expect(configuration.getProperty("server.tmp.dir")).andReturn(Configuration.SERVER_TMP_DIR_DEFAULT); //!!!! Map<String,String> props = new HashMap<String, String>(); props.put(Configuration.HIVE_METASTORE_PASSWORD_PROPERTY, "pass"); @@ -261,7 +263,8 @@ public class ClientConfigResourceProviderTest { expect(Runtime.getRuntime()).andReturn(runtime); expect(mockFile.exists()).andReturn(true); expect(runtime.exec("ambari-python-wrap /tmp/stacks/S1/V1/PIG/package/null generate_configs null " + - "/tmp/stacks/S1/V1/PIG/package /tmp/ambari-server/structured-out.json INFO /tmp/ambari-server")) + "/tmp/stacks/S1/V1/PIG/package /var/lib/ambari-server/tmp/structured-out.json " + + "INFO /var/lib/ambari-server/tmp")) .andReturn(process).once(); // replay @@ -278,6 +281,7 @@ public class ClientConfigResourceProviderTest { verify(managementController, clusters, cluster, ambariMetaInfo, stackId, componentInfo,commandScriptDefinition, clusterConfig, host, service, serviceComponent, serviceComponentHost, serviceInfo, configHelper, runtime, process); + PowerMock.verifyAll(); } @Test @@ -295,8 +299,8 @@ public class ClientConfigResourceProviderTest { PropertyHelper.getKeyPropertyIds(type), managementController); - Predicate predicate = new PredicateBuilder().property(ClientConfigResourceProvider.COMPONENT_COMPONENT_NAME_PROPERTY_ID).equals("HDFS_CLIENT"). - toPredicate(); + Predicate predicate = new PredicateBuilder().property( + ClientConfigResourceProvider.COMPONENT_COMPONENT_NAME_PROPERTY_ID).equals("HDFS_CLIENT").toPredicate(); try { provider.deleteResources(predicate); Assert.fail("Expected an UnsupportedOperationException"); http://git-wip-us.apache.org/repos/asf/ambari/blob/adb87a44/ambari-server/src/test/python/TestBootstrap.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/TestBootstrap.py b/ambari-server/src/test/python/TestBootstrap.py index d511222..6bcc94f 100644 --- a/ambari-server/src/test/python/TestBootstrap.py +++ b/ambari-server/src/test/python/TestBootstrap.py @@ -109,8 +109,8 @@ class TestBootstrap(TestCase): utime = 1234 bootstrap_obj.getUtime = MagicMock(return_value=utime) ret = bootstrap_obj.getRunSetupWithPasswordCommand("hostname") - expected = "sudo -S python /tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \ - "ambariServer 8440 < /tmp/host_pass{0}".format(utime) + expected = "sudo -S python /var/lib/ambari-agent/data/tmp/setupAgent{0}.py hostname TEST_PASSPHRASE " \ + "ambariServer 8440 < /var/lib/ambari-agent/data/tmp/host_pass{0}".format(utime) self.assertEquals(ret, expected) @@ -343,6 +343,25 @@ class TestBootstrap(TestCase): self.assertEquals(rf, "/etc/yum.repos.d/ambari.repo") + @patch.object(SSH, "__init__") + @patch.object(SSH, "run") + @patch.object(HostLog, "write") + def test_createTargetDir(self, write_mock, run_mock, + init_mock): + shared_state = SharedState("root", "sshkey_file", "scriptDir", "bootdir", + "setupAgentFile", "ambariServer", "centos6", + None, "8440") + bootstrap_obj = Bootstrap("hostname", shared_state) + expected = 42 + init_mock.return_value = None + run_mock.return_value = expected + res = bootstrap_obj.createTargetDir() + self.assertEquals(res, expected) + command = str(init_mock.call_args[0][3]) + self.assertEqual(command, + "[ -d /var/lib/ambari-agent/data/tmp ] || sudo mkdir -p /var/lib/ambari-agent/data/tmp ; " + "sudo chown root /var/lib/ambari-agent/data/tmp") + @patch.object(Bootstrap, "getOsCheckScript") @patch.object(Bootstrap, "getOsCheckScriptRemoteLocation") @patch.object(SCP, "__init__") @@ -460,7 +479,7 @@ class TestBootstrap(TestCase): command = str(init_mock.call_args[0][3]) self.assertEqual(command, "chmod a+x OsCheckScriptRemoteLocation && " - "env PYTHONPATH=$PYTHONPATH:/tmp OsCheckScriptRemoteLocation centos6") + "env PYTHONPATH=$PYTHONPATH:/var/lib/ambari-agent/data/tmp OsCheckScriptRemoteLocation centos6") @patch.object(SSH, "__init__") @@ -678,7 +697,7 @@ class TestBootstrap(TestCase): hasPassword_mock.return_value = False try_to_execute_mock.return_value = {"exitstatus": 0, "log":"log0", "errormsg":"errormsg0"} bootstrap_obj.run() - self.assertEqual(try_to_execute_mock.call_count, 6) # <- Adjust if changed + self.assertEqual(try_to_execute_mock.call_count, 7) # <- Adjust if changed self.assertTrue(createDoneFile_mock.called) self.assertEqual(bootstrap_obj.getStatus()["return_code"], 0) @@ -689,7 +708,7 @@ class TestBootstrap(TestCase): hasPassword_mock.return_value = True try_to_execute_mock.return_value = {"exitstatus": 0, "log":"log0", "errormsg":"errormsg0"} bootstrap_obj.run() - self.assertEqual(try_to_execute_mock.call_count, 9) # <- Adjust if changed + self.assertEqual(try_to_execute_mock.call_count, 10) # <- Adjust if changed self.assertTrue(createDoneFile_mock.called) self.assertEqual(bootstrap_obj.getStatus()["return_code"], 0)
