SLIDER-160: agent functional tests upload binaries/ini file if there is a difference
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5464584d Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5464584d Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5464584d Branch: refs/heads/feature/SLIDER-151_Implement_full_slider_API_in_REST_and_switch_client_to_it Commit: 5464584d784cf57348bf22100cee2581d5a887c0 Parents: 8327e52 Author: Steve Loughran <ste...@apache.org> Authored: Tue Jun 24 17:40:49 2014 -0700 Committer: Steve Loughran <ste...@apache.org> Committed: Tue Jun 24 17:40:49 2014 -0700 ---------------------------------------------------------------------- .../funtest/framework/AgentUploads.groovy | 41 +++++++++++++++--- .../funtest/framework/CommandTestBase.groovy | 9 ++-- .../funtest/framework/FileUploader.groovy | 45 ++++++++++++++------ .../funtest/framework/FuntestProperties.groovy | 12 ++++-- .../lifecycle/AgentCommandTestBase.groovy | 39 ++++++----------- .../lifecycle/TestClusterBuildDestroy.groovy | 2 +- 6 files changed, 94 insertions(+), 54 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy index b1c29c0..5f10c0e 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/AgentUploads.groovy @@ -20,22 +20,51 @@ package org.apache.slider.funtest.framework import groovy.util.logging.Slf4j import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.FileUtil import org.apache.hadoop.fs.Path -import org.apache.hadoop.fs.permission.FsPermission import org.apache.hadoop.security.UserGroupInformation +import org.apache.hadoop.fs.FileSystem as HadoopFS @Slf4j -class AgentUploads { +class AgentUploads implements FuntestProperties { final Configuration conf + private final FileUploader uploader + private final HadoopFS clusterFS + private final Path homeDir + AgentUploads(Configuration conf) { this.conf = conf + uploader = new FileUploader(conf, UserGroupInformation.currentUser) + clusterFS = uploader.fileSystem + homeDir = clusterFS.homeDirectory } - def execUploadSequence() { - UserGroupInformation.loginUserFromKeytabAndReturnUGI("yarn") + /** + * Upload agent-related files + * @param tarballDir + * @param force + * @return + */ + def uploadAgentFiles(File tarballDir, boolean force) { + def localAgentTar = new File(tarballDir, AGENT_SLIDER_GZ_IN_SLIDER_TAR) + assert localAgentTar.exists() + + def agentTarballPath = new Path( + homeDir, + AGENT_TAR_FILENAME) + + // Upload the agent tarball + uploader.copyIfOutOfDate(localAgentTar, agentTarballPath, force) + + File localAgentIni = new File(tarballDir, AGENT_INI_IN_SLIDER_TAR) + // Upload the agent.ini + def agentIniPath = new Path(homeDir, AGENT_INI) + uploader.copyIfOutOfDate(localAgentIni, agentIniPath, force) + + return [agentTarballPath, agentIniPath] + + } - + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy index d2accbb..61ae804 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy @@ -50,17 +50,16 @@ abstract class CommandTestBase extends SliderTestUtils { LoggerFactory.getLogger(CommandTestBase.class); public static final String SLIDER_CONF_DIR = sysprop(SLIDER_CONF_DIR_PROP) - public static final String SLIDER_BIN_DIR = sysprop(SLIDER_BIN_DIR_PROP) - public static final File SLIDER_BIN_DIRECTORY = new File( - SLIDER_BIN_DIR).canonicalFile + public static final String SLIDER_TAR_DIR = sysprop(SLIDER_BIN_DIR_PROP) + public static final File SLIDER_TAR_DIRECTORY = new File( + SLIDER_TAR_DIR).canonicalFile public static final File SLIDER_SCRIPT = new File( - SLIDER_BIN_DIRECTORY, + SLIDER_TAR_DIRECTORY, BIN_SLIDER).canonicalFile public static final File SLIDER_CONF_DIRECTORY = new File( SLIDER_CONF_DIR).canonicalFile public static final File SLIDER_CONF_XML = new File(SLIDER_CONF_DIRECTORY, CLIENT_CONFIG_FILENAME).canonicalFile - public static final YarnConfiguration SLIDER_CONFIG public static final int THAW_WAIT_TIME public static final int FREEZE_WAIT_TIME http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy index dd42473..262693c 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FileUploader.groovy @@ -23,8 +23,9 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.FileUtil import org.apache.hadoop.fs.Path import org.apache.hadoop.fs.permission.FsPermission +import org.apache.hadoop.security.AccessControlException import org.apache.hadoop.security.UserGroupInformation - +import org.apache.hadoop.fs.FileSystem as HadoopFS @Slf4j class FileUploader { final Configuration conf @@ -38,39 +39,59 @@ class FileUploader { /** * Copy if the file is considered out of date * @param src - * @param dest + * @param destPath * @param force * @return */ - public boolean copyIfOutOfDate(File src, Path dest, boolean force) { + public boolean copyIfOutOfDate(File src, Path destPath, boolean force) { def srcLen = src.length() - def fs = getFileSystem(user, dest.toUri()) + def fs = getFileSystem(destPath) boolean toCopy = force if (!toCopy) { try { - def status = fs.getFileStatus(dest) + def status = fs.getFileStatus(destPath) toCopy = status.len != srcLen } catch (FileNotFoundException fnfe) { toCopy = true; } } if (toCopy) { - log.info("Copying $src to $dest") - fs.mkdirs(dest, FsPermission.dirDefault) - return FileUtil.copy(src, fs, dest, false, conf) + log.info("Copying $src to $destPath") + try { + fs.delete(destPath, true) + fs.mkdirs(destPath.getParent(), FsPermission.dirDefault) + return FileUtil.copy(src, fs, destPath, false, conf) + } catch (AccessControlException ace) { + log.error("No write access to test user home directory. " + + "Ensure home directory exists and has correct permissions." + + ace, ace) + throw ace + } } else { - log.debug("Skipping copy as the destination $dest considered up to date") + log.debug("Skipping copy as the destination $destPath considered up to date") return false; } + } + public HadoopFS getFileSystem(Path dest) { + getFileSystem(user, dest) + } + + public HadoopFS getFileSystem() { + getFileSystem(user, HadoopFS.getDefaultUri(conf)) } - public static def getFileSystem( + public def getFileSystem( + UserGroupInformation user, final Path path) { + return getFileSystem(user, path.toUri()) + + } + public def getFileSystem( UserGroupInformation user, final URI uri) { + SudoClosure.sudo(user) { - org.apache.hadoop.fs.FileSystem.get(uri, conf); + HadoopFS.get(uri, conf); } - } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy index 8cbc098..9b63c22 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy @@ -57,8 +57,12 @@ public interface FuntestProperties extends SliderXMLConfKeysForTesting { String ENV_SLIDER_CLASSPATH_EXTRA = "SLIDER_CLASSPATH_EXTRA" String SCRIPT_NAME = "slider" - static final String KEY_TEST_CONF_XML = "slider.test.conf.xml" - static final String KEY_TEST_CONF_DIR = "slider.test.conf.dir" - static final String BIN_SLIDER = "bin/slider" - static final String AGENT_SLIDER_GZ = "agent/slider-agent.tar.gz" + String KEY_TEST_CONF_XML = "slider.test.conf.xml" + String KEY_TEST_CONF_DIR = "slider.test.conf.dir" + String BIN_SLIDER = "bin/slider" + String AGENT_INI = "agent.ini" + String AGENT_INI_IN_SLIDER_TAR = "agent/conf/" + AGENT_INI + + String AGENT_TAR_FILENAME = "slider-agent.tar.gz" + String AGENT_SLIDER_GZ_IN_SLIDER_TAR = "agent/" + AGENT_TAR_FILENAME } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy index 6333fad..c3d6bd6 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy @@ -20,10 +20,10 @@ package org.apache.slider.funtest.lifecycle import groovy.util.logging.Slf4j import org.apache.hadoop.fs.Path -import org.apache.hadoop.security.AccessControlException import org.apache.slider.common.SliderExitCodes import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions +import org.apache.slider.funtest.framework.AgentUploads import org.apache.slider.funtest.framework.CommandTestBase import org.apache.slider.funtest.framework.FuntestProperties import org.apache.slider.funtest.framework.SliderShell @@ -53,37 +53,29 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { static { AGENTTESTS_ENABLED = SLIDER_CONFIG.getBoolean(KEY_TEST_AGENT_ENABLED, false) - LOCAL_SLIDER_AGENT_TARGZ = new File( - SLIDER_BIN_DIRECTORY, - AGENT_SLIDER_GZ).canonicalFile LOCAL_AGENT_CONF = new File(AGENT_CONF).canonicalFile } @Rule public TemporaryFolder folder = new TemporaryFolder(); + public static void assumeAgentTestsEnabled() { + assumeFunctionalTestsEnabled() + assume(AGENTTESTS_ENABLED, "Agent tests disabled") + } + @BeforeClass public static void setupAgent() { assumeAgentTestsEnabled() - try { - // Upload the agent tarball - assume(LOCAL_SLIDER_AGENT_TARGZ.exists(), "Slider agent not found at $LOCAL_SLIDER_AGENT_TARGZ") - agentTarballPath = new Path(clusterFS.homeDirectory, "slider-agent.tar.gz") - Path localTarball = new Path(LOCAL_SLIDER_AGENT_TARGZ.toURI()); - clusterFS.copyFromLocalFile(false, true, localTarball, agentTarballPath) - - // Upload the agent.ini - assume(LOCAL_AGENT_CONF.exists(), "Agent config not found at $LOCAL_AGENT_CONF") - agtIniPath = new Path(clusterFS.homeDirectory, "agent.ini") - Path localAgtIni = new Path(LOCAL_AGENT_CONF.toURI()); - clusterFS.copyFromLocalFile(false, true, localAgtIni, agtIniPath) - } catch (AccessControlException ace) { - log.info "No write access to test user home directory. " + - "Ensure home directory exists and has correct permissions." + ace.getMessage() - fail("Ensure home directory exists and has correct permissions for test user.") - } } + + @Before + public void uploadAgentTarball() { + def agentUploads = new AgentUploads(SLIDER_CONFIG) + (agentTarballPath, agtIniPath) = + agentUploads.uploadAgentFiles(SLIDER_TAR_DIRECTORY, false) + } @Before @@ -105,11 +97,6 @@ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions { clusterFS.copyFromLocalFile(false, true, localAppPkg, appPkgPath) } - public static void assumeAgentTestsEnabled() { - assumeFunctionalTestsEnabled() - assume(AGENTTESTS_ENABLED, "Agent tests disabled") - } - public static void logShell(SliderShell shell) { for (String str in shell.out) { log.info str http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5464584d/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy index 3db6c5b..1907a2c 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy @@ -42,7 +42,7 @@ public class TestClusterBuildDestroy extends AgentCommandTestBase @BeforeClass public static void prepareCluster() { - assumeFunctionalTestsEnabled(); + setupCluster(CLUSTER) }