Repository: oozie Updated Branches: refs/heads/master 6323a8e43 -> fe2da6e57
http://git-wip-us.apache.org/repos/asf/oozie/blob/fe2da6e5/sharelib/git/src/main/java/org/apache/oozie/action/hadoop/GitOperations.java ---------------------------------------------------------------------- diff --git a/sharelib/git/src/main/java/org/apache/oozie/action/hadoop/GitOperations.java b/sharelib/git/src/main/java/org/apache/oozie/action/hadoop/GitOperations.java new file mode 100644 index 0000000..3505790 --- /dev/null +++ b/sharelib/git/src/main/java/org/apache/oozie/action/hadoop/GitOperations.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.eclipse.jgit.api.CloneCommand; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.TransportConfigCallback; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.transport.JschConfigSessionFactory; +import org.eclipse.jgit.transport.OpenSshConfig; +import org.eclipse.jgit.transport.SshSessionFactory; +import org.eclipse.jgit.transport.SshTransport; +import org.eclipse.jgit.transport.Transport; +import org.eclipse.jgit.util.FS; + +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; + +class GitOperations { + private final URI srcURL; + private final String branch; + private final File credentialFile; + + GitOperations(final URI gitSrc, final String branch, final File credentialFile) { + this.srcURL = gitSrc; + this.branch = branch; + this.credentialFile = credentialFile; + } + + /** + * Used by GitOperations to wrap a Throwable when an Exception occurs + */ + @SuppressWarnings("serial") + static class GitOperationsException extends Exception { + GitOperationsException(final String message, Throwable throwable) { + super(message, throwable); + } + } + + /** + * Clones a Git repository + * @param outputDir location in which to clone the Git repository + * @throws GitOperationsException if the Git clone fails + */ + private void cloneRepo(final File outputDir) throws GitOperationsException { + final SshSessionFactory sshSessionFactory = new JschConfigSessionFactory() { + @Override + protected void configure(final OpenSshConfig.Host host, final Session session) { + // nop + } + + @Override + protected JSch createDefaultJSch(final FS fs) throws JSchException { + JSch.setConfig("StrictHostKeyChecking", "no"); + final JSch defaultJSch = super.createDefaultJSch(fs); + + if (credentialFile != null) { + defaultJSch.addIdentity(credentialFile.toString()); + } + + return defaultJSch; + } + }; + + final CloneCommand cloneCommand = Git.cloneRepository(); + cloneCommand.setURI(srcURL.toString()); + + if (srcURL.getScheme().toLowerCase().equals("ssh")) { + cloneCommand.setTransportConfigCallback(new TransportConfigCallback() { + @Override + public void configure(final Transport transport) { + final SshTransport sshTransport = (SshTransport)transport; + sshTransport.setSshSessionFactory(sshSessionFactory); + } + }); + } + + cloneCommand.setDirectory(outputDir); + // set our branch identifier + if (branch != null) { + cloneCommand.setBranchesToClone(Arrays.asList("refs/heads/" + branch)); + } + + try { + cloneCommand.call(); + } catch (final GitAPIException e) { + throw new GitOperationsException("Unable to clone Git repo: ", e); + } + } + + /** + * Clone a Git repo up to a FileSystem + * + * @param destination - Hadoop FileSystem path to which repository should be cloned + * @throws GitOperationsException if the Git operations fail + * @throws IOException if the HDFS or local file system operations fail + */ + @SuppressFBWarnings(value ="PATH_TRAVERSAL_IN", justification = "Path is created runtime") + String cloneRepoToFS(final Path destination) throws IOException, GitOperationsException { + final File tempDir = GitMain.createTempDir("git"); + + final Configuration conf = new Configuration(); + final FileSystem fs = FileSystem.get(destination.toUri(), conf); + + cloneRepo(tempDir); + + // create a list of files and directories to upload + final File src = new File(tempDir.getAbsolutePath()); + final ArrayList<Path> srcs = new ArrayList<Path>(1000); + final File[] sourceFiles = src.listFiles(); + if (sourceFiles != null) { + for (final File sourceFile : sourceFiles) { + srcs.add(new Path(sourceFile.toString())); + } + } + + System.out.println("Finished cloning to local"); + + fs.mkdirs(destination); + fs.copyFromLocalFile(false, true, srcs.toArray(new Path[0]), destination); + + System.out.println("Finished copying to " + destination.toString()); + + return destination.toString(); + } + +} http://git-wip-us.apache.org/repos/asf/oozie/blob/fe2da6e5/sharelib/git/src/test/java/org/apache/oozie/action/hadoop/GitServer.java ---------------------------------------------------------------------- diff --git a/sharelib/git/src/test/java/org/apache/oozie/action/hadoop/GitServer.java b/sharelib/git/src/test/java/org/apache/oozie/action/hadoop/GitServer.java new file mode 100644 index 0000000..03d6e55 --- /dev/null +++ b/sharelib/git/src/test/java/org/apache/oozie/action/hadoop/GitServer.java @@ -0,0 +1,155 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import org.apache.commons.io.FileUtils; + +import org.apache.oozie.util.XLog; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.errors.RepositoryNotFoundException; +import org.eclipse.jgit.lib.Repository; +import org.eclipse.jgit.storage.file.FileRepositoryBuilder; +import org.eclipse.jgit.transport.Daemon; +import org.eclipse.jgit.transport.DaemonClient; +import org.eclipse.jgit.transport.ServiceMayNotContinueException; +import org.eclipse.jgit.transport.resolver.RepositoryResolver; +import org.eclipse.jgit.transport.resolver.ServiceNotAuthorizedException; +import org.eclipse.jgit.transport.resolver.ServiceNotEnabledException; + +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +class GitServer { + private static final XLog LOG = XLog.getLog(GitServer.class); + + /** + * A simple git server serving anynymous git: protocol + */ + private final Map<String, Repository> repositories = new HashMap<>(); + private Daemon server; + private final int localPort; + + GitServer() throws IOException { + LOG.info("Creating Git server"); + + this.localPort = findAvailablePort(); + + LOG.info("Git server created, port {0} will be used", this.localPort); + } + + void start() throws IOException { + LOG.info("Starting Git server on port {0}", this.localPort); + + this.server = new Daemon(new InetSocketAddress(this.localPort)); + this.server.getService("git-receive-pack").setEnabled(true); + this.server.setRepositoryResolver(new EmptyRepositoryResolverImplementation()); + this.server.start(); + + LOG.info("Git server started"); + } + + int getLocalPort() { + return localPort; + } + + private int findAvailablePort() throws IOException { + try (final ServerSocket serverSocket = new ServerSocket(0)) { + final int availablePort = serverSocket.getLocalPort(); + LOG.info("Found available port {0}", availablePort); + return availablePort; + } + } + + void stopAndCleanupReposServer() { + cleanUpRepos(); + this.server.stop(); + } + + /** + * A method to: + * <ul> + * <li>remove all files on disk for all repositories</li> + * <li>clear the repositories listed for the {@link GitServer}</li> + * </ul> + */ + private void cleanUpRepos() { + for (final Repository repository : repositories.values()) { + final File workTree = repository.getWorkTree(); + try { + FileUtils.deleteDirectory(workTree.getParentFile()); + } + catch (final IOException e) { + LOG.warn("Could not delete parent directory of working tree: ", e); + } + } + repositories.clear(); + } + + /** + * A simple class RepositoryResolver to provide an empty repository for non-existant repo requests + */ + private final class EmptyRepositoryResolverImplementation implements + RepositoryResolver<DaemonClient> { + + @Override + public Repository open(final DaemonClient client, final String name) { + Repository repo = repositories.get(name); + if (repo == null) { + try { + final Path workDir = Files.createTempDirectory("GitTestSetup"); + //git init + repo = FileRepositoryBuilder.create(new File(workDir.resolve(name).toFile(), ".git")); + repo.create(); + // commit into the filesystem + final Git git = new Git(repo); + // one needs an initial commit for a proper clone + addEmptyCommit(git); + git.close(); + // serve the Git repo + repositories.put(name, repo); + } + catch (final Exception e) { + throw new RuntimeException(); + } + } + return repo; + } + + /** + * Add an empty commit to a Git repository + * @param git a Git repository to add an empty commit to + */ + void addEmptyCommit(final Git git) { + try { + git.commit().setMessage("Empty commit").call(); + } + catch (final GitAPIException e) { + throw new RuntimeException(e); + } + } + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/fe2da6e5/sharelib/git/src/test/java/org/apache/oozie/action/hadoop/TestGitActionExecutor.java ---------------------------------------------------------------------- diff --git a/sharelib/git/src/test/java/org/apache/oozie/action/hadoop/TestGitActionExecutor.java b/sharelib/git/src/test/java/org/apache/oozie/action/hadoop/TestGitActionExecutor.java new file mode 100644 index 0000000..433f7e2 --- /dev/null +++ b/sharelib/git/src/test/java/org/apache/oozie/action/hadoop/TestGitActionExecutor.java @@ -0,0 +1,213 @@ +/** * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import java.io.IOException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.AclEntry; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.oozie.action.ActionExecutorException; +import org.apache.oozie.service.WorkflowAppService; +import org.apache.oozie.util.XConfiguration; +import org.apache.oozie.util.XmlUtils; +import org.apache.oozie.WorkflowActionBean; +import org.apache.oozie.WorkflowJobBean; +import org.jdom.Element; +import org.junit.Assert; + +public class TestGitActionExecutor extends ActionExecutorTestCase { + + @SuppressWarnings("unchecked") + public void testWhenParametersFilledThenConfigurationFieldsPopulated() throws Exception { + final GitActionExecutor ae = new GitActionExecutor(); + assertTrue("Can not find GitMain class in launcher classes", + ae.getLauncherClasses().contains(GitMain.class)); + + final Path testKey = new Path(getAppPath().toString() + "/test_key"); + createTestFile(testKey); + final FileSystem fs = getFileSystem(); + fs.setPermission(testKey, FsPermission.valueOf("-r--------")); + + final String repoUrl = "https://github.com/apache/oozie";; + final String keyUrl = testKey.toString(); + final String destDir = "repoDir"; + final String branch = "myBranch"; + final Element actionXml = XmlUtils.parseXml("<git>" + + "<resource-manager>" + getJobTrackerUri() + "</resource-manager>" + + "<name-node>" + getNameNodeUri() + "</name-node>" + + "<git-uri>" + repoUrl + "</git-uri>"+ + "<branch>" + branch + "</branch>"+ + "<key-path>" + keyUrl + "</key-path>"+ + "<destination-uri>" + destDir + "</destination-uri>" + + "</git>"); + + final XConfiguration protoConf = new XConfiguration(); + protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); + + final WorkflowJobBean wf = createBaseWorkflow(protoConf, GitActionExecutor.GIT_ACTION_TYPE + "-action"); + final WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); + action.setType(ae.getType()); + + final Context context = new Context(wf, action); + final Configuration conf = ae.createBaseHadoopConf(context, actionXml); + ae.setupActionConf(conf, context, actionXml, getFsTestCaseDir()); + + assertEquals("git uri must be set", repoUrl, conf.get(GitActionExecutor.GIT_URI)); + assertEquals("key path must be set", keyUrl, conf.get(GitActionExecutor.KEY_PATH)); + assertEquals("branch must be set", branch, conf.get(GitActionExecutor.GIT_BRANCH)); + assertEquals("destination uri must be set", destDir, conf.get(GitActionExecutor.DESTINATION_URI)); + } + + public void testAccessKeyPermissionsInsecure() throws Exception { + final GitActionExecutor ae = new GitActionExecutor(); + + final Path testKey = new Path(getAppPath().toString() + "/test_key"); + createTestFile(testKey); + final FileSystem fs = getFileSystem(); + fs.setPermission(testKey, FsPermission.valueOf("-r-----rw-")); + + final String repoUrl = "https://github.com/apache/oozie";; + final String keyUrl = testKey.toString(); + final String destDir = "repoDir"; + final String branch = "myBranch"; + final Element actionXml = XmlUtils.parseXml("<git>" + + "<resource-manager>" + getJobTrackerUri() + "</resource-manager>" + + "<name-node>" + getNameNodeUri() + "</name-node>" + + "<git-uri>" + repoUrl + "</git-uri>"+ + "<branch>" + branch + "</branch>"+ + "<key-path>" + keyUrl + "</key-path>"+ + "<destination-uri>" + destDir + "</destination-uri>" + + "</git>"); + + final XConfiguration protoConf = new XConfiguration(); + protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); + + final WorkflowJobBean wf = createBaseWorkflow(protoConf, GitActionExecutor.GIT_ACTION_TYPE + "-action"); + final WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); + action.setType(ae.getType()); + + final Context context = new Context(wf, action); + final Configuration conf = ae.createBaseHadoopConf(context, actionXml); + try { + // we expect this to throw an ActionExecutorException: + ae.setupActionConf(conf, context, actionXml, getFsTestCaseDir()); + Assert.fail("Expected ActionExecutorException"); + } + catch (final ActionExecutorException e) { + if (!e.getMessage().contains("insecure")) { + Assert.fail("Unexpected exception message: " + e.getMessage()); + } + } + } + + public void testAccessKeyACLsSecure() throws Exception { + final GitActionExecutor ae = new GitActionExecutor(); + + final Path testKey = new Path(getAppPath().toString() + "/test_key"); + createTestFile(testKey); + // set file permissions to be secure -- allowing only the owner to read + final FileSystem fs = getFileSystem(); + fs.setPermission(testKey, FsPermission.valueOf("-r--------")); + fs.setAcl(testKey, AclEntry.parseAclSpec("user::rwx,user:foo:rw-,group::r--,other::---", true)); + + final String repoUrl = "https://github.com/apache/oozie";; + final String keyUrl = testKey.toString(); + final String destDir = "repoDir"; + final String branch = "myBranch"; + final Element actionXml = XmlUtils.parseXml("<git>" + + "<resource-manager>" + getJobTrackerUri() + "</resource-manager>" + + "<name-node>" + getNameNodeUri() + "</name-node>" + + "<git-uri>" + repoUrl + "</git-uri>"+ + "<branch>" + branch + "</branch>"+ + "<key-path>" + keyUrl + "</key-path>"+ + "<destination-uri>" + destDir + "</destination-uri>" + + "</git>"); + + final XConfiguration protoConf = new XConfiguration(); + protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); + + final WorkflowJobBean wf = createBaseWorkflow(protoConf, GitActionExecutor.GIT_ACTION_TYPE + "-action"); + final WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); + action.setType(ae.getType()); + + final Context context = new Context(wf, action); + final Configuration conf = ae.createBaseHadoopConf(context, actionXml); + try { + ae.setupActionConf(conf, context, actionXml, getFsTestCaseDir()); + } + catch (final ActionExecutorException e) { + fail("Unexpected exception, could not check ACLs: " + e.getMessage()); + } + } + + public void testAccessKeyACLsInsecure() throws Exception { + final GitActionExecutor ae = new GitActionExecutor(); + + final Path testKey = new Path(getAppPath().toString() + "/test_key"); + createTestFile(testKey); + // set file permissions to be secure -- allowing only the owner to read + final FileSystem fs = getFileSystem(); + fs.setPermission(testKey, FsPermission.valueOf("-r--------")); + fs.setAcl(testKey, AclEntry.parseAclSpec("user::rwx,user:foo:rw-,group::r--,other::r--", true)); + + final String repoUrl = "https://github.com/apache/oozie";; + final String keyUrl = testKey.toString(); + final String destDir = "repoDir"; + final String branch = "myBranch"; + final Element actionXml = XmlUtils.parseXml("<git>" + + "<resource-manager>" + getJobTrackerUri() + "</resource-manager>" + + "<name-node>" + getNameNodeUri() + "</name-node>" + + "<git-uri>" + repoUrl + "</git-uri>"+ + "<branch>" + branch + "</branch>"+ + "<key-path>" + keyUrl + "</key-path>"+ + "<destination-uri>" + destDir + "</destination-uri>" + + "</git>"); + + final XConfiguration protoConf = new XConfiguration(); + protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); + + final WorkflowJobBean wf = createBaseWorkflow(protoConf, GitActionExecutor.GIT_ACTION_TYPE + "-action"); + final WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); + action.setType(ae.getType()); + + final Context context = new Context(wf, action); + + try { + ae.createBaseHadoopConf(context, actionXml); + } + catch (final Exception e) { + fail("Unexpected exception, could not create Hadoop configuration with insecure setup: " + e.getMessage()); + } + } + + private void createTestFile(final Path testFile) throws IOException { + final FileSystem fs = getFileSystem(); + final FSDataOutputStream file = fs.create(testFile); + file.writeUTF(""); + file.close(); + } + + @Override + protected void setSystemProps() throws Exception { + super.setSystemProps(); + setSystemProperty("oozie.service.ActionService.executor.classes", GitActionExecutor.class.getName()); + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/fe2da6e5/sharelib/git/src/test/java/org/apache/oozie/action/hadoop/TestGitMain.java ---------------------------------------------------------------------- diff --git a/sharelib/git/src/test/java/org/apache/oozie/action/hadoop/TestGitMain.java b/sharelib/git/src/test/java/org/apache/oozie/action/hadoop/TestGitMain.java new file mode 100644 index 0000000..3b236d4 --- /dev/null +++ b/sharelib/git/src/test/java/org/apache/oozie/action/hadoop/TestGitMain.java @@ -0,0 +1,119 @@ +/** * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import org.apache.commons.io.FileUtils; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.oozie.client.OozieClient; +import org.apache.oozie.client.WorkflowJob; +import org.apache.oozie.fluentjob.api.action.GitAction; +import org.apache.oozie.fluentjob.api.action.GitActionBuilder; +import org.apache.oozie.fluentjob.api.factory.WorkflowFactory; +import org.apache.oozie.fluentjob.api.workflow.Workflow; +import org.apache.oozie.fluentjob.api.workflow.WorkflowBuilder; +import org.apache.oozie.test.MiniOozieTestCase; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Properties; + +public class TestGitMain extends MiniOozieTestCase { + + private GitMain gitMain; + + @Override + protected void setUp() throws Exception { + super.setUp(); + + gitMain = new GitMain(); + gitMain.setNameNode(getFileSystem().getUri().toString()); + } + + public void testGitRepoMustHaveScheme() throws Exception { + final OozieClient client = this.getClient(); + final Properties conf = client.createConfiguration(); + + final class GitWorkflowFactory implements WorkflowFactory { + @Override + public Workflow create() { + final GitAction gitAction = GitActionBuilder.create() + .withName("git-action") + .withResourceManager(getJobTrackerUri()) + .withNameNode(getNameNodeUri()) + .withConfigProperty("mapred.job.queue.name", "default") + //Note: no URI scheme + .withGitUri("aURLWithoutAScheme/andSomeMorePathStuff") + .withDestinationUri("repoDir") + .build(); + + final Workflow gitWorkflow = new WorkflowBuilder() + .withName("git-workflow") + .withDagContainingNode(gitAction).build(); + + return gitWorkflow; + } + } + final Path workflowUri = new Path(getFsTestCaseDir().toString(), "workflow.xml"); + conf.setProperty(OozieClient.APP_PATH, workflowUri.toString()); + + writeHDFSFile(workflowUri, new GitWorkflowFactory().create().asXml()); + + final String jobId = client.run(conf); + + waitFor(60_000, new Predicate() { + @Override + public boolean evaluate() throws Exception { + return client.getJobInfo(jobId).getStatus() != WorkflowJob.Status.RUNNING; + } + }); + + // Should fail with something akin to: + // Failing Oozie Launcher, Action Configuration does not have a proper URI repoDir null scheme. + // org.apache.oozie.action.hadoop.GitMain$GitMainException: Action Configuration does not have + // a proper URI repoDir null scheme. + // One needs to look into the YARN logs though to see this as client.getJobLog() is not implemented + assertEquals("Git action should have been killed", + WorkflowJob.Status.KILLED, + client.getJobInfo(jobId).getStatus()); + } + + public void testGitKeyFileIsCopiedToHDFS() throws Exception { + final Path credentialFilePath = Path.mergePaths(getFsTestCaseDir(), new Path("/key_dir/my_key.dsa")); + final String credentialFileData = "Key file data"; + Path.mergePaths(getFsTestCaseDir(), new Path("/destDir")); + + writeHDFSFile(credentialFilePath, credentialFileData); + + final File localFile = gitMain.getKeyFromFS(credentialFilePath); + final String testOutput = new String(Files.readAllBytes(localFile.toPath())); + + assertTrue("credential file length mismatch", credentialFileData.length() > 0); + assertEquals("credential file data mismatch", credentialFileData, testOutput); + + FileUtils.deleteDirectory(new File(localFile.getParent())); + } + + private void writeHDFSFile(final Path hdfsFilePath, final String fileData) throws IOException { + try (final FSDataOutputStream hdfsFileOS = getFileSystem().create(hdfsFilePath)) { + hdfsFileOS.write(fileData.getBytes()); + hdfsFileOS.flush(); + } + } +} http://git-wip-us.apache.org/repos/asf/oozie/blob/fe2da6e5/sharelib/git/src/test/java/org/apache/oozie/action/hadoop/TestIntegrationGitActionExecutor.java ---------------------------------------------------------------------- diff --git a/sharelib/git/src/test/java/org/apache/oozie/action/hadoop/TestIntegrationGitActionExecutor.java b/sharelib/git/src/test/java/org/apache/oozie/action/hadoop/TestIntegrationGitActionExecutor.java new file mode 100644 index 0000000..20368da --- /dev/null +++ b/sharelib/git/src/test/java/org/apache/oozie/action/hadoop/TestIntegrationGitActionExecutor.java @@ -0,0 +1,133 @@ + +/** * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.action.hadoop; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.commons.io.IOUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.WorkflowActionBean; +import org.apache.oozie.WorkflowJobBean; +import org.apache.oozie.client.WorkflowAction; +import org.apache.oozie.service.WorkflowAppService; +import org.apache.oozie.util.XConfiguration; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Map; + +public class TestIntegrationGitActionExecutor extends ActionExecutorTestCase{ + + @Override + protected void setSystemProps() throws Exception { + super.setSystemProps(); + setSystemProperty("oozie.service.ActionService.executor.classes", GitActionExecutor.class.getName()); + } + + public void testWhenRepoIsClonedThenGitIndexContentIsReadSuccessfully() throws Exception { + final Path outputPath = getFsTestCaseDir(); + final Path gitRepo = Path.mergePaths(outputPath, new Path("/repoDir")); + final Path gitIndex = Path.mergePaths(gitRepo, new Path("/.git/config")); + + final GitServer gitServer = new GitServer(); + + final String localRepo = String.format("git://127.0.0.1:%s/repo.git", gitServer.getLocalPort()); + final String actionXml = "<git>" + + "<resource-manager>" + getJobTrackerUri() + "</resource-manager>" + + "<name-node>" + getNameNodeUri() + "</name-node>" + + "<git-uri>" + localRepo + "</git-uri>"+ + "<destination-uri>" + gitRepo + "</destination-uri>" + + "</git>"; + + final Context context = createContext(actionXml); + final String launcherId = submitAction(context); + + try { + gitServer.start(); + + waitUntilYarnAppDoneAndAssertSuccess(launcherId); + } + finally { + gitServer.stopAndCleanupReposServer(); + } + final Map<String, String> actionData = LauncherHelper.getActionData(getFileSystem(), context.getActionDir(), + context.getProtoActionConf()); + assertFalse(LauncherHelper.hasIdSwap(actionData)); + + final GitActionExecutor ae = new GitActionExecutor(); + ae.check(context, context.getAction()); + assertEquals("launcherId and action.externalId should be the same", launcherId, context.getAction().getExternalId()); + assertEquals("action should have been SUCCEEDED", "SUCCEEDED", context.getAction().getExternalStatus()); + + ae.end(context, context.getAction()); + assertEquals("action.status should be OK", WorkflowAction.Status.OK, context.getAction().getStatus()); + + assertTrue("could not create test case output path", getFileSystem().exists(outputPath)); + assertTrue("could not save git index", getFileSystem().exists(gitIndex)); + + try (final InputStream is = getFileSystem().open(gitIndex)) { + final String gitIndexContent = IOUtils.toString(is, StandardCharsets.UTF_8); + + assertTrue("could not read git index", gitIndexContent.toLowerCase().contains("core")); + assertTrue("could not read git index", gitIndexContent.toLowerCase().contains("remote")); + } + } + + private Context createContext(final String actionXml) throws Exception { + final GitActionExecutor ae = new GitActionExecutor(); + + final XConfiguration protoConf = new XConfiguration(); + protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser()); + + final FileSystem fs = getFileSystem(); + SharelibUtils.addToDistributedCache(GitActionExecutor.GIT_ACTION_TYPE, fs, getFsTestCaseDir(), protoConf); + + final WorkflowJobBean wf = createBaseWorkflow(protoConf, GitActionExecutor.GIT_ACTION_TYPE + "-action"); + final WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0); + action.setType(ae.getType()); + action.setConf(actionXml); + + return new Context(wf, action); + } + + private String submitAction(final Context context) throws Exception { + final GitActionExecutor ae = new GitActionExecutor(); + + final WorkflowAction action = context.getAction(); + + ae.prepareActionDir(getFileSystem(), context); + ae.submitLauncher(getFileSystem(), context, action); + + final String externalId = action.getExternalId(); + final String trackerUri = action.getTrackerUri(); + final String consoleUrl = action.getConsoleUrl(); + assertNotNull("action.externalId should be filled", externalId); + assertNotNull("action.trackerUri should be filled", trackerUri); + assertNotNull("action.consoleUrl should be filled", consoleUrl); + + final Configuration conf = createJobConf(); + + final String runningJobExternalId = context.getAction().getExternalId(); + + assertNotNull("running job has a valid externalId", runningJobExternalId); + + return runningJobExternalId; + } + +} http://git-wip-us.apache.org/repos/asf/oozie/blob/fe2da6e5/sharelib/pom.xml ---------------------------------------------------------------------- diff --git a/sharelib/pom.xml b/sharelib/pom.xml index 39cea25..1c51bfe 100644 --- a/sharelib/pom.xml +++ b/sharelib/pom.xml @@ -35,6 +35,7 @@ <module>streaming</module> <module>hcatalog</module> <module>pig</module> + <module>git</module> <module>hive</module> <module>hive2</module> <module>sqoop</module> http://git-wip-us.apache.org/repos/asf/oozie/blob/fe2da6e5/src/main/assemblies/sharelib.xml ---------------------------------------------------------------------- diff --git a/src/main/assemblies/sharelib.xml b/src/main/assemblies/sharelib.xml index 07dc69c..8ce558c 100644 --- a/src/main/assemblies/sharelib.xml +++ b/src/main/assemblies/sharelib.xml @@ -66,6 +66,10 @@ <directory>${basedir}/spark/target/partial-sharelib</directory> <outputDirectory>/</outputDirectory> </fileSet> + <fileSet> + <directory>${basedir}/git/target/partial-sharelib</directory> + <outputDirectory>/</outputDirectory> + </fileSet> </fileSets> </assembly> http://git-wip-us.apache.org/repos/asf/oozie/blob/fe2da6e5/webapp/pom.xml ---------------------------------------------------------------------- diff --git a/webapp/pom.xml b/webapp/pom.xml index fd3f89f..938b0c9 100644 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -229,6 +229,10 @@ <groupId>org.apache.oozie</groupId> <artifactId>oozie-sharelib-spark</artifactId> </artifactItem> + <artifactItem> + <groupId>org.apache.oozie</groupId> + <artifactId>oozie-sharelib-git</artifactId> + </artifactItem> </artifactItems> <outputDirectory>${project.build.directory}/oozie-webapp-${project.version}/WEB-INF/lib</outputDirectory> <overWriteIfNewer>true</overWriteIfNewer>
