Repository: incubator-falcon Updated Branches: refs/heads/master 437109646 -> 8fc5ff611
FALCON-861 Add ACL tests for falcon client and ACL update. Contributed by Raghav Kumar Gautam Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/8fc5ff61 Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/8fc5ff61 Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/8fc5ff61 Branch: refs/heads/master Commit: 8fc5ff611439d632217f689581769df1a39df377 Parents: 4371096 Author: Raghav Kumar Gautam <rag...@apache.org> Authored: Wed Nov 5 09:26:11 2014 -0800 Committer: Raghav Kumar Gautam <rag...@apache.org> Committed: Wed Nov 5 09:26:11 2014 -0800 ---------------------------------------------------------------------- falcon-regression/CHANGES.txt | 3 + falcon-regression/README.md | 7 + falcon-regression/merlin-core/pom.xml | 5 + .../core/enumsAndConstants/MerlinConstants.java | 15 +- .../core/interfaces/FalconClientBuilder.java | 178 +++++++++++++++++++ .../core/interfaces/IEntityManagerHelper.java | 33 ++++ .../core/supportClasses/ExecResult.java | 25 ++- .../falcon/regression/core/util/AssertUtil.java | 13 ++ .../falcon/regression/core/util/ExecUtil.java | 77 +++----- .../falcon/regression/core/util/FileUtil.java | 47 +++++ .../falcon/regression/core/util/OSUtil.java | 3 + .../regression/security/AclValidationTest.java | 2 +- .../regression/security/ClusterAclTest.java | 2 +- .../regression/security/FalconClientTest.java | 106 +++++++++++ .../falcon/regression/security/FeedAclTest.java | 43 ++++- .../regression/security/ProcessAclTest.java | 54 +++++- falcon-regression/pom.xml | 5 + 17 files changed, 550 insertions(+), 68 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8fc5ff61/falcon-regression/CHANGES.txt ---------------------------------------------------------------------- diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt index 2854075..0f88e29 100644 --- a/falcon-regression/CHANGES.txt +++ b/falcon-regression/CHANGES.txt @@ -5,6 +5,9 @@ Trunk (Unreleased) INCOMPATIBLE CHANGES NEW FEATURES + FALCON-861 Add ACL tests for falcon client and ACL update + (Raghav Kumar Gautam) + FALCON-844 List instances tests (Paul Isaychuk via Ruslan Ostafiychuk) FALCON-841 Test falcon process with different frequencies http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8fc5ff61/falcon-regression/README.md ---------------------------------------------------------------------- diff --git a/falcon-regression/README.md b/falcon-regression/README.md index 12b15dd..abfa689 100644 --- a/falcon-regression/README.md +++ b/falcon-regression/README.md @@ -126,6 +126,13 @@ For testing with kerberos set keytabs properties for different users: falcon.super2.user.keytab=/home/qa/hadoopqa/keytabs/falcon2.headless.keytab other.user.keytab=/home/qa/hadoopqa/keytabs/root.headless.keytab +Testing on Windows +------------------ +Some tests switch user to run commands as a different user. Location of binary to switch user is +configurable: + + windows.su.binary=ExecuteAs.exe + Automatic capture of oozie logs ------------------------------- For full falcon regression runs. It might be desirable to pull all oozie job http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8fc5ff61/falcon-regression/merlin-core/pom.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/pom.xml b/falcon-regression/merlin-core/pom.xml index 21641a3..a9699b9 100644 --- a/falcon-regression/merlin-core/pom.xml +++ b/falcon-regression/merlin-core/pom.xml @@ -159,5 +159,10 @@ <groupId>org.apache.falcon</groupId> <artifactId>falcon-client</artifactId> </dependency> + + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8fc5ff61/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java index dab5d2c..d6780f3 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java @@ -46,7 +46,10 @@ public final class MerlinConstants { Config.getProperty("current_user.group.name", "users"); /** a user that does not belong to the group of current user. */ - public static final String DIFFERENT_USER = Config.getProperty("other.user.name", "root"); + public static final String DIFFERENT_USER_NAME = Config.getProperty("other.user.name", "root"); + + /** a user that does not belong to the group of current user. */ + public static final String DIFFERENT_USER_GROUP = Config.getProperty("other.user.name", "root"); /** falcon super user. */ public static final String FALCON_SUPER_USER_NAME = @@ -59,6 +62,7 @@ public final class MerlinConstants { private static final String USER_2_KEYTAB_STR = "user2_keytab"; public static final String USER2_NAME; private static HashMap<String, String> keyTabMap; + private static HashMap<String, String> passwordMap; public static final String ACL_OWNER = Config.getProperty("ACL.OWNER", RequestKeys.CURRENT_USER); public static final String ACL_GROUP = Config.getProperty("ACL.GROUP", "default"); public static final String USER_REALM = Config.getProperty("USER.REALM", ""); @@ -81,11 +85,18 @@ public final class MerlinConstants { keyTabMap.put(user2Name, user2Keytab); keyTabMap.put(FALCON_SUPER_USER_NAME, Config.getProperty("falcon.super.user.keytab")); keyTabMap.put(FALCON_SUPER_USER2_NAME, Config.getProperty("falcon.super.user2.keytab")); - keyTabMap.put(DIFFERENT_USER, Config.getProperty("other.user.keytab")); + keyTabMap.put(DIFFERENT_USER_NAME, Config.getProperty("other.user.keytab")); + passwordMap = new HashMap<String, String>(); + passwordMap.put(DIFFERENT_USER_NAME, Config.getProperty("other.user.password")); } public static String getKeytabForUser(String user) { Assert.assertTrue(keyTabMap.containsKey(user), "Unknown user: " + user); return keyTabMap.get(user); } + + public static String getPasswordForUser(String user) { + Assert.assertTrue(passwordMap.containsKey(user), "Unknown user: " + user); + return passwordMap.get(user); + } } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8fc5ff61/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/FalconClientBuilder.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/FalconClientBuilder.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/FalconClientBuilder.java new file mode 100644 index 0000000..02b7529 --- /dev/null +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/FalconClientBuilder.java @@ -0,0 +1,178 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.falcon.regression.core.interfaces; + +import org.apache.commons.exec.CommandLine; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.builder.Builder; +import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; +import org.apache.falcon.regression.core.util.Config; +import org.apache.falcon.regression.core.util.OSUtil; + +import java.util.ArrayList; +import java.util.List; + +/** + * FalconClientBuilder is to be used for launching falcon client command. + */ +public class FalconClientBuilder implements Builder<CommandLine> { + private final String user; + private final CommandLine commandLine; + private final List<String> args; + private final SuType suType; + + private enum SuType { + /** + * Takes care of switching user on linux. Current implemented through sudo. + */ + LIN_SUDO { + @Override + public CommandLine getCommandLine(String user) { + return CommandLine.parse("sudo").addArgument("-u") + .addArgument(user).addArgument(falconClientBinary); + } + @Override + public void addArgsToCommandLine(CommandLine commandLine, List<String> args) { + for (String arg : args) { + commandLine.addArgument(arg); + } + } + }, + /** + * Takes care of switching user on windows. Needs to be implemented. + */ + WIN_SU { + @Override + public CommandLine getCommandLine(String user) { + return CommandLine.parse(OSUtil.WIN_SU_BINARY) + .addArgument("-u").addArgument(user) + .addArgument("-p").addArgument(MerlinConstants.getPasswordForUser(user)) + .addArgument(falconClientBinary); + } + @Override + public void addArgsToCommandLine(CommandLine commandLine, List<String> args) { + String lastArg = StringUtils.join(args, " "); + commandLine.addArgument(lastArg, true); + } + }, + /** + * Takes care of the case where no user switch is required. + */ + NONE { + @Override + public CommandLine getCommandLine(String user) { + return CommandLine.parse(falconClientBinary); + } + @Override + public void addArgsToCommandLine(CommandLine commandLine, List<String> args) { + for (String arg : args) { + commandLine.addArgument(arg); + } + } + }; + + private static final String falconClientBinary = + Config.getProperty("falcon.client.binary", "falcon"); + public abstract void addArgsToCommandLine(CommandLine commandLine, List<String> args); + public abstract CommandLine getCommandLine(String user); + } + + private FalconClientBuilder(String user) { + this.user = user; + args = new ArrayList<String>(); + if (user == null) { + suType = SuType.NONE; + commandLine = suType.getCommandLine(null); + } else { + if (OSUtil.IS_WINDOWS) { + suType = SuType.WIN_SU; + commandLine = suType.getCommandLine(user); + } else { + suType = SuType.LIN_SUDO; + //attempting sudo su - root -c "falcon admin -version" + commandLine = suType.getCommandLine(user); + } + } + } + + /** + * Get an instance of FalconClientBuilder + * @return instance of FalconClientBuilder + */ + public static FalconClientBuilder getBuilder() { + return new FalconClientBuilder(null); + } + + /** + * Get an instance of FalconClientBuilder for the given user. It would do commandline + * construction in a way that the final command is run as given user. + * @return instance of FalconClientBuilder + */ + public static FalconClientBuilder getBuilder(String user) { + return new FalconClientBuilder(user); + } + + /** + * Add the given argument + * @param arg argument to be added to builder + * @return this + */ + private FalconClientBuilder addArg(String arg) { + args.add(arg); + return this; + } + + /** + * Create submit command + * @param entityType type of the entity + * @param fileName file containing the entity to be submitted + * @return this + */ + public FalconClientBuilder getSubmitCommand(String entityType, String fileName) { + addArg("entity").addArg("-submit"); + addArg("-type").addArg(entityType); + addArg("-file").addArg(fileName); + return this; + } + + /** + * Create delete command + * @param entityType type of the entity + * @param entityName name of the entity to be deleted + * @return this + */ + public FalconClientBuilder getDeleteCommand(String entityType, String entityName) { + addArg("entity").addArg("-delete"); + addArg("-type").addArg(entityType); + addArg("-name").addArg(entityName); + return this; + } + + + /** + * Build the CommandLine object for this FalconClientBuilder + * @return instance of CommandLine object + */ + @Override + public CommandLine build() { + suType.addArgsToCommandLine(commandLine, args); + return new CommandLine(commandLine); + } +} http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8fc5ff61/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/IEntityManagerHelper.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/IEntityManagerHelper.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/IEntityManagerHelper.java index 7264142..e1d808e 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/IEntityManagerHelper.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/interfaces/IEntityManagerHelper.java @@ -19,12 +19,15 @@ package org.apache.falcon.regression.core.interfaces; import com.jcraft.jsch.JSchException; +import org.apache.commons.exec.CommandLine; import org.apache.commons.lang.exception.ExceptionUtils; import org.apache.falcon.regression.core.response.InstancesSummaryResult; import org.apache.falcon.regression.core.response.InstancesResult; import org.apache.falcon.regression.core.response.ServiceResponse; +import org.apache.falcon.regression.core.supportClasses.ExecResult; import org.apache.falcon.regression.core.util.Config; import org.apache.falcon.regression.core.util.ExecUtil; +import org.apache.falcon.regression.core.util.FileUtil; import org.apache.falcon.regression.core.util.HCatUtil; import org.apache.falcon.regression.core.util.InstanceUtil; import org.apache.falcon.regression.core.util.OSUtil; @@ -538,6 +541,36 @@ public abstract class IEntityManagerHelper { } /** + * Submit an entity through falcon client. + * @param entityStr string of the entity to be submitted + * @return + * @throws IOException + */ + public ExecResult clientSubmit(final String entityStr) throws IOException { + LOGGER.info("Submitting " + getEntityType() + " through falcon client: \n" + + Util.prettyPrintXml(entityStr)); + final String fileName = FileUtil.writeEntityToFile(entityStr); + final CommandLine commandLine = FalconClientBuilder.getBuilder() + .getSubmitCommand(getEntityType(), fileName).build(); + return ExecUtil.executeCommand(commandLine); + } + + /** + * Delete an entity through falcon client. + * @param entityStr string of the entity to be submitted + * @return + * @throws IOException + */ + public ExecResult clientDelete(final String entityStr, String user) throws IOException { + final String entityName = getEntityName(entityStr); + LOGGER.info("Deleting " + getEntityType() + ": " + entityName); + final CommandLine commandLine = FalconClientBuilder.getBuilder(user) + .getDeleteCommand(getEntityType(), entityName).build(); + return ExecUtil.executeCommand(commandLine); + } + + + /** * Retrieves entities summary. * @param clusterName compulsory parameter for request * @param params list of optional parameters http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8fc5ff61/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/supportClasses/ExecResult.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/supportClasses/ExecResult.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/supportClasses/ExecResult.java index f878aa8..c5172eb 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/supportClasses/ExecResult.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/supportClasses/ExecResult.java @@ -18,22 +18,31 @@ package org.apache.falcon.regression.core.supportClasses; +import org.apache.commons.exec.CommandLine; + public final class ExecResult { private final int exitVal; private final String output; private final String error; + private final CommandLine commandLine; - public ExecResult(final int exitVal, final String output, final String error) { + public ExecResult(CommandLine commandLine, final int exitVal, final String output, + final String error) { this.exitVal = exitVal; this.output = output; this.error = error; + this.commandLine = commandLine; } public int getExitVal() { return exitVal; } + public boolean hasSuceeded() { + return exitVal == 0; + } + public String getOutput() { return output; } @@ -41,4 +50,18 @@ public final class ExecResult { public String getError() { return error; } + + public CommandLine getCommandLine() { + return commandLine; + } + + @Override + public String toString() { + return "ExecResult{" + + "exitVal=" + exitVal + + ", output='" + output + '\'' + + ", error='" + error + '\'' + + ", commandLine=" + commandLine + + '}'; + } } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8fc5ff61/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java index 569d1f3..3384af0 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/AssertUtil.java @@ -24,6 +24,7 @@ import org.apache.falcon.regression.core.bundle.Bundle; import org.apache.falcon.regression.core.response.APIResult; import org.apache.falcon.regression.core.response.InstancesResult; import org.apache.falcon.regression.core.response.ServiceResponse; +import org.apache.falcon.regression.core.supportClasses.ExecResult; import org.apache.hadoop.fs.ContentSummary; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -207,6 +208,18 @@ public final class AssertUtil { } /** + * Assert that command executed unsuccessfully + * + * @param execResult ExecResult of the command execution + */ + public static void assertFailed(ExecResult execResult, String expectedMessage) { + Assert.assertFalse(execResult.hasSuceeded(), + "Unexpectedly succeeded execResult: " + execResult); + Assert.assertTrue(execResult.getError().contains(expectedMessage), + "Expected error: " + expectedMessage + " in execResult: " + execResult); + } + + /** * Checks that ServiceResponse status is status FAILED with some status code. * * @param response ServiceResponse http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8fc5ff61/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/ExecUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/ExecUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/ExecUtil.java index d240e76..8dcb202 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/ExecUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/ExecUtil.java @@ -23,11 +23,16 @@ import com.jcraft.jsch.JSch; import com.jcraft.jsch.JSchException; import com.jcraft.jsch.Session; import com.jcraft.jsch.UserInfo; -import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.ExecuteWatchdog; +import org.apache.commons.exec.PumpStreamHandler; +import org.apache.falcon.regression.core.supportClasses.ExecResult; import org.apache.log4j.Logger; import org.testng.Assert; import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -132,37 +137,29 @@ public final class ExecUtil { } public static ExecResult executeCommand(String command) { - LOGGER.info("Command to be executed: " + command); - StringBuilder errors = new StringBuilder(); - StringBuilder output = new StringBuilder(); + return executeCommand(CommandLine.parse(command)); + } + public static ExecResult executeCommand(CommandLine commandLine) { + LOGGER.info("Command to be executed: " + commandLine); + DefaultExecutor executor = new DefaultExecutor(); + executor.setWatchdog(new ExecuteWatchdog(5 * 1000)); //timeout of 5 seconds + final ByteArrayOutputStream outStream = new ByteArrayOutputStream(); + final ByteArrayOutputStream errStream = new ByteArrayOutputStream(); + executor.setStreamHandler(new PumpStreamHandler(outStream, errStream)); + int exitVal = 1; try { - Process process = Runtime.getRuntime().exec(command); - - BufferedReader errorReader = - new BufferedReader(new InputStreamReader(process.getErrorStream())); - BufferedReader consoleReader = - new BufferedReader(new InputStreamReader(process.getInputStream())); - - String line; - while ((line = errorReader.readLine()) != null) { - errors.append(line).append("\n"); - } - - while ((line = consoleReader.readLine()) != null) { - output.append(line).append("\n"); - } - final int exitVal = process.waitFor(); - LOGGER.info("exitVal: " + exitVal); - LOGGER.info("output: " + output); - LOGGER.info("errors: " + errors); - return new ExecResult(exitVal, output.toString().trim(), errors.toString().trim()); - } catch (InterruptedException e) { - Assert.fail("Process execution failed:" + ExceptionUtils.getStackTrace(e)); + exitVal = executor.execute(commandLine); } catch (IOException e) { - Assert.fail("Process execution failed:" + ExceptionUtils.getStackTrace(e)); + LOGGER.warn("Caught exception: " + e); } - return null; + final String output = outStream.toString(); + final String errors = errStream.toString(); + + LOGGER.info("exitVal: " + exitVal); + LOGGER.info("output: " + output); + LOGGER.info("errors: " + errors); + return new ExecResult(commandLine, exitVal, output.trim(), errors.trim()); } public static int executeCommandGetExitCode(String command) { @@ -206,28 +203,4 @@ public final class ExecUtil { } } - private static final class ExecResult { - - private final int exitVal; - private final String output; - private final String error; - - private ExecResult(final int exitVal, final String output, final String error) { - this.exitVal = exitVal; - this.output = output; - this.error = error; - } - - public int getExitVal() { - return exitVal; - } - - public String getOutput() { - return output; - } - - public String getError() { - return error; - } - } } http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8fc5ff61/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/FileUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/FileUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/FileUtil.java new file mode 100644 index 0000000..1a97e1d --- /dev/null +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/FileUtil.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.falcon.regression.core.util; + +import org.apache.commons.io.FileUtils; +import org.apache.log4j.Logger; + +import java.io.File; +import java.io.IOException; + +public class FileUtil { + private static final Logger logger = Logger.getLogger(FileUtil.class); + private FileUtil() { + } + + /** + * Writes an entity to a file and returns the filename. + * @param entity to be written + * @return name of the file + * @throws IOException + */ + public static String writeEntityToFile(String entity) throws IOException { + final String entityName = Util.readEntityName(entity); + final File entityFile = new File(entityName + ".xml"); + logger.info("attempting to write: " + entityName + " at location " + + entityFile.getAbsolutePath()); + FileUtils.write(entityFile, entity); + return entityFile.getAbsolutePath(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8fc5ff61/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java index 86d4d47..ed29d07 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OSUtil.java @@ -29,6 +29,9 @@ public final class OSUtil { } public static final boolean IS_WINDOWS = System.getProperty("os.name").toLowerCase().startsWith("windows"); + public static final String WIN_SU_BINARY = + Config.getProperty("windows.su.binary", "ExecuteAs.exe"); + public static final String SEPARATOR = System.getProperty("file.separator", "/"); public static final String RESOURCES = String.format("src%stest%sresources%s", SEPARATOR, SEPARATOR, SEPARATOR); http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8fc5ff61/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/AclValidationTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/AclValidationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/AclValidationTest.java index d7f21b7..d0568d7 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/AclValidationTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/AclValidationTest.java @@ -130,7 +130,7 @@ public class AclValidationTest extends BaseTestClass { public Object[][] generateUserAndGroup() { return new Object[][] { {MerlinConstants.USER2_NAME, MerlinConstants.CURRENT_USER_GROUP}, - {MerlinConstants.CURRENT_USER_NAME, MerlinConstants.DIFFERENT_USER}, + {MerlinConstants.CURRENT_USER_NAME, MerlinConstants.DIFFERENT_USER_NAME}, {MerlinConstants.CURRENT_USER_NAME, "nonexistinggroup"}, {"nonexistinguser", MerlinConstants.CURRENT_USER_GROUP}, }; http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8fc5ff61/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ClusterAclTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ClusterAclTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ClusterAclTest.java index affff56..74d9ccb 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ClusterAclTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ClusterAclTest.java @@ -102,7 +102,7 @@ public class ClusterAclTest extends BaseTestClass { ); final Object[][] notAllowedCombinations = MathUtil.crossProduct( - new String[]{MerlinConstants.DIFFERENT_USER}, + new String[]{MerlinConstants.DIFFERENT_USER_NAME}, new EntityOp[]{EntityOp.dependency, EntityOp.listing, EntityOp.definition}, new Boolean[]{false} ); http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8fc5ff61/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java new file mode 100644 index 0000000..c18da3d --- /dev/null +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.falcon.regression.security; + +import org.apache.falcon.regression.core.bundle.Bundle; +import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; +import org.apache.falcon.regression.core.helpers.ColoHelper; +import org.apache.falcon.regression.core.supportClasses.ExecResult; +import org.apache.falcon.regression.core.util.AssertUtil; +import org.apache.falcon.regression.core.util.BundleUtil; +import org.apache.falcon.regression.core.util.HadoopUtil; +import org.apache.falcon.regression.core.util.KerberosHelper; +import org.apache.falcon.regression.core.util.OSUtil; +import org.apache.falcon.regression.testHelper.BaseTestClass; +import org.apache.hadoop.fs.FileSystem; +import org.apache.log4j.Logger; +import org.testng.annotations.AfterClass; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.io.IOException; +import java.lang.reflect.Method; + +@Test(groups = "authorization") +public class FalconClientTest extends BaseTestClass { + private static final Logger LOGGER = Logger.getLogger(AclValidationTest.class); + + private ColoHelper cluster = servers.get(0); + private FileSystem clusterFS = serverFS.get(0); + private String baseTestDir = baseHDFSDir + "/AuthorizationTest"; + private String aggregateWorkflowDir = baseTestDir + "/aggregator"; + private String feedInputPath = baseTestDir + "/input" + MINUTE_DATE_PATTERN; + + @BeforeClass(alwaysRun = true) + public void uploadWorkflow() throws Exception { + HadoopUtil.uploadDir(clusterFS, aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE); + } + + @BeforeMethod(alwaysRun = true) + public void setup(Method method) throws Exception { + LOGGER.info("test name: " + method.getName()); + Bundle bundle = BundleUtil.readELBundle(); + bundles[0] = new Bundle(bundle, cluster); + bundles[0].generateUniqueBundle(); + bundles[0].setInputFeedDataPath(feedInputPath); + bundles[0].setProcessWorkflow(aggregateWorkflowDir); + } + + /** + * Test error thrown by falcon client, when acl of the submitted cluster has bad values + * @throws Exception + */ + @Test + public void badClusterSubmit() throws Exception { + bundles[0].setCLusterACL(MerlinConstants.DIFFERENT_USER_NAME, + MerlinConstants.CURRENT_USER_GROUP, "*"); + final String cluster = bundles[0].getClusters().get(0); + final ExecResult execResult = prism.getClusterHelper().clientSubmit(cluster); + AssertUtil.assertFailed(execResult, "cluster submission failed"); + } + + /** + * Test error thrown by falcon client, a user tries to delete a cluster that it should not be + * able to delete + * @throws Exception + */ + @Test + public void badClusterDelete() throws Exception { + bundles[0].submitClusters(prism); + final String cluster = bundles[0].getClusters().get(0); + KerberosHelper.loginFromKeytab(MerlinConstants.DIFFERENT_USER_NAME); + final ExecResult execResult = + prism.getClusterHelper().clientDelete(cluster, MerlinConstants.DIFFERENT_USER_NAME); + AssertUtil.assertFailed(execResult, "cluster deletion failed"); + } + + @AfterMethod(alwaysRun = true) + public void tearDown() { + KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); + removeBundles(); + } + + @AfterClass(alwaysRun = true) + public void tearDownClass() throws IOException { + cleanTestDirs(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8fc5ff61/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FeedAclTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FeedAclTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FeedAclTest.java index 79e5db6..6767af3 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FeedAclTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FeedAclTest.java @@ -24,6 +24,7 @@ import org.apache.falcon.regression.core.bundle.Bundle; import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; import org.apache.falcon.regression.core.helpers.ColoHelper; import org.apache.falcon.regression.core.interfaces.IEntityManagerHelper; +import org.apache.falcon.regression.core.util.AssertUtil; import org.apache.falcon.regression.core.util.BundleUtil; import org.apache.falcon.regression.core.util.HadoopUtil; import org.apache.falcon.regression.core.util.KerberosHelper; @@ -104,7 +105,7 @@ public class FeedAclTest extends BaseTestClass { ); final Object[][] notAllowedCombinations = MathUtil.crossProduct( - new String[]{MerlinConstants.DIFFERENT_USER}, + new String[]{MerlinConstants.DIFFERENT_USER_NAME}, falconReadOps, new Boolean[]{false} ); @@ -146,7 +147,7 @@ public class FeedAclTest extends BaseTestClass { ); final Object[][] notAllowedCombinations = MathUtil.crossProduct( - new String[]{MerlinConstants.DIFFERENT_USER}, + new String[]{MerlinConstants.DIFFERENT_USER_NAME}, falconEditOps, new Boolean[]{false} ); @@ -189,7 +190,7 @@ public class FeedAclTest extends BaseTestClass { ); final Object[][] notAllowedCombinations = MathUtil.crossProduct( - new String[]{MerlinConstants.DIFFERENT_USER}, + new String[]{MerlinConstants.DIFFERENT_USER_NAME}, new EntityOp[]{EntityOp.delete, EntityOp.update, EntityOp.schedule, EntityOp.submitAndSchedule, EntityOp.suspend, EntityOp.resume}, new Boolean[]{false} @@ -198,6 +199,42 @@ public class FeedAclTest extends BaseTestClass { return MathUtil.append(allowedCombinations, notAllowedCombinations); } + /** + * Test feed acl modification. + * @throws Exception + */ + @Test + public void feedAclUpdate() throws Exception { + bundles[0].submitClusters(prism); + final String oldFeed = bundles[0].getInputFeedFromBundle(); + AssertUtil.assertSucceeded(feedHelper.submitAndSchedule(oldFeed)); + final FeedMerlin feedMerlin = new FeedMerlin(oldFeed); + feedMerlin.setACL(MerlinConstants.DIFFERENT_USER_NAME, + MerlinConstants.DIFFERENT_USER_GROUP, "*"); + final String newFeed = feedMerlin.toString(); + AssertUtil.assertSucceeded(feedHelper.update(oldFeed, newFeed)); + //check that current user can't access the feed + for(EntityOp op : new EntityOp[]{EntityOp.status, EntityOp.dependency, EntityOp.listing, + EntityOp.definition}) { + final boolean executeRes = + op.executeAs(MerlinConstants.DIFFERENT_USER_NAME, feedHelper, newFeed); + Assert.assertFalse(executeRes, "Unexpected result: user " + + MerlinConstants.DIFFERENT_USER_GROUP + " was not able to perform: " + op); + } + //check that different user can access the feed + KerberosHelper.loginFromKeytab(MerlinConstants.DIFFERENT_USER_NAME); + for(EntityOp op : new EntityOp[]{EntityOp.status, EntityOp.dependency, EntityOp.listing, + EntityOp.definition}) { + final boolean executeRes = + op.executeAs(MerlinConstants.DIFFERENT_USER_NAME, feedHelper, newFeed); + Assert.assertTrue(executeRes, "Unexpected result: user " + + MerlinConstants.DIFFERENT_USER_GROUP + " was not able to perform: " + op); + } + //check modification permissions + AssertUtil.assertSucceeded(feedHelper.update(newFeed, oldFeed, + MerlinConstants.DIFFERENT_USER_NAME)); + } + @AfterMethod(alwaysRun = true) public void tearDown() { KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8fc5ff61/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ProcessAclTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ProcessAclTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ProcessAclTest.java index f0775e7..bc79835 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ProcessAclTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/ProcessAclTest.java @@ -24,6 +24,7 @@ import org.apache.falcon.regression.core.bundle.Bundle; import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; import org.apache.falcon.regression.core.helpers.ColoHelper; import org.apache.falcon.regression.core.interfaces.IEntityManagerHelper; +import org.apache.falcon.regression.core.util.AssertUtil; import org.apache.falcon.regression.core.util.BundleUtil; import org.apache.falcon.regression.core.util.HadoopUtil; import org.apache.falcon.regression.core.util.KerberosHelper; @@ -88,7 +89,7 @@ public class ProcessAclTest extends BaseTestClass { bundles[0].submitProcess(true); final boolean executeRes = op.executeAs(user, processHelper, processString); Assert.assertEquals(executeRes, isAllowed, "Unexpected result user " + user + - " performing: " + op); + " performing: " + op); } @DataProvider(name = "generateUserReadOpsPermissions") @@ -96,14 +97,15 @@ public class ProcessAclTest extends BaseTestClass { final EntityOp[] falconReadOps = {EntityOp.status, EntityOp.dependency, EntityOp.listing, EntityOp.definition}; final Object[][] allowedCombinations = MathUtil.crossProduct( - new String[]{MerlinConstants.FALCON_SUPER_USER_NAME, MerlinConstants.FALCON_SUPER_USER2_NAME, - MerlinConstants.USER2_NAME}, + new String[]{MerlinConstants.FALCON_SUPER_USER_NAME, + MerlinConstants.FALCON_SUPER_USER2_NAME, + MerlinConstants.USER2_NAME}, falconReadOps, new Boolean[]{true} ); final Object[][] notAllowedCombinations = MathUtil.crossProduct( - new String[]{MerlinConstants.DIFFERENT_USER}, + new String[]{MerlinConstants.DIFFERENT_USER_NAME}, falconReadOps, new Boolean[]{false} ); @@ -145,9 +147,9 @@ public class ProcessAclTest extends BaseTestClass { ); final Object[][] notAllowedCombinations = MathUtil.crossProduct( - new String[]{MerlinConstants.DIFFERENT_USER}, - falconEditOps, - new Boolean[]{false} + new String[]{MerlinConstants.DIFFERENT_USER_NAME}, + falconEditOps, + new Boolean[]{false} ); return MathUtil.append(allowedCombinations, notAllowedCombinations); @@ -187,7 +189,7 @@ public class ProcessAclTest extends BaseTestClass { ); final Object[][] notAllowedCombinations = MathUtil.crossProduct( - new String[]{MerlinConstants.DIFFERENT_USER}, + new String[]{MerlinConstants.DIFFERENT_USER_NAME}, new EntityOp[]{EntityOp.delete, EntityOp.update, EntityOp.schedule, EntityOp.submitAndSchedule, EntityOp.suspend, EntityOp.resume}, new Boolean[]{false} @@ -196,6 +198,42 @@ public class ProcessAclTest extends BaseTestClass { return MathUtil.append(allowedCombinations, notAllowedCombinations); } + /** + * Test process acl modification. + * @throws Exception + */ + @Test + public void processAclUpdate() throws Exception { + bundles[0].submitFeedsScheduleProcess(); + final String oldProcess = bundles[0].getProcessData(); + AssertUtil.assertSucceeded(prism.getProcessHelper().submitAndSchedule(oldProcess)); + final ProcessMerlin processMerlin = new ProcessMerlin(oldProcess); + processMerlin.setACL(MerlinConstants.DIFFERENT_USER_NAME, + MerlinConstants.DIFFERENT_USER_GROUP, "*"); + final String newProcess = processMerlin.toString(); + AssertUtil.assertSucceeded(processHelper.update(oldProcess, newProcess)); + //check that current user can't access the feed + for(EntityOp op : new EntityOp[]{EntityOp.status, EntityOp.dependency, EntityOp.listing, + EntityOp.definition}) { + final boolean executeRes = + op.executeAs(MerlinConstants.DIFFERENT_USER_NAME, processHelper, newProcess); + Assert.assertFalse(executeRes, "Unexpected result: user " + + MerlinConstants.DIFFERENT_USER_GROUP + " was not able to perform: " + op); + } + //check that different user can access the feed + KerberosHelper.loginFromKeytab(MerlinConstants.DIFFERENT_USER_NAME); + for(EntityOp op : new EntityOp[]{EntityOp.status, EntityOp.dependency, EntityOp.listing, + EntityOp.definition}) { + final boolean executeRes = + op.executeAs(MerlinConstants.DIFFERENT_USER_NAME, processHelper, newProcess); + Assert.assertTrue(executeRes, "Unexpected result: user " + + MerlinConstants.DIFFERENT_USER_GROUP + " was not able to perform: " + op); + } + //check modification permissions + AssertUtil.assertSucceeded(processHelper.update(newProcess, oldProcess, + MerlinConstants.DIFFERENT_USER_NAME)); + } + @AfterMethod(alwaysRun = true) public void tearDown() { KerberosHelper.loginFromKeytab(MerlinConstants.CURRENT_USER_NAME); http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/8fc5ff61/falcon-regression/pom.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/pom.xml b/falcon-regression/pom.xml index 0424cb8..1179843 100644 --- a/falcon-regression/pom.xml +++ b/falcon-regression/pom.xml @@ -220,6 +220,11 @@ <artifactId>selenium-support</artifactId> <version>2.41.0</version> </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-exec</artifactId> + <version>1.2</version> + </dependency> </dependencies> </dependencyManagement>
