Add ProcessUpdateTest, PipelineInstanceDependencyTest and other tests and test fixes. Contributed by Raghav Gautam and Paul Isaychuk
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/9e6d5a6c Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/9e6d5a6c Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/9e6d5a6c Branch: refs/heads/master Commit: 9e6d5a6c58d99479cc58c11b5b05b11bf8d90821 Parents: 5a55bae Author: Paul Isaychuk <[email protected]> Authored: Thu Oct 15 14:15:58 2015 +0300 Committer: Paul Isaychuk <[email protected]> Committed: Tue Oct 20 11:46:31 2015 +0300 ---------------------------------------------------------------------- falcon-regression/CHANGES.txt | 3 + .../regression/Entities/ProcessMerlin.java | 9 + .../helpers/entity/AbstractEntityHelper.java | 10 + .../regression/core/util/EntityLineageUtil.java | 63 ++++ .../regression/core/util/InstanceUtil.java | 40 ++- .../regression/core/util/KerberosHelper.java | 9 + .../falcon/regression/core/util/OozieUtil.java | 52 +++ .../falcon/regression/core/util/Util.java | 22 ++ .../ui/search/AbstractSearchPage.java | 69 +++- .../regression/ui/search/ClusterWizardPage.java | 40 +-- .../regression/ui/search/EntityWizardPage.java | 94 ++++++ .../regression/ui/search/FeedWizardPage.java | 27 +- .../falcon/regression/ui/search/LoginPage.java | 1 + .../regression/ui/search/MirrorWizardPage.java | 13 +- .../falcon/regression/ui/search/PageHeader.java | 30 +- .../regression/ui/search/ProcessWizardPage.java | 37 +- .../falcon/regression/ExternalFSTest.java | 2 +- .../falcon/regression/FeedReplicationTest.java | 109 +++++- .../regression/ProcessInstanceStatusTest.java | 39 +++ .../falcon/regression/ProcessUpdateTest.java | 112 +++++++ .../falcon/regression/TestngListener.java | 2 +- .../falcon/regression/hive/dr/HiveDRTest.java | 118 ++++--- .../falcon/regression/hive/dr/HiveDbDRTest.java | 61 ++-- .../regression/hive/dr/RecipeExecLocation.java | 63 ++++ .../lineage/ListProcessInstancesTest.java | 27 +- .../regression/searchUI/ClusterSetupTest.java | 48 ++- .../regression/searchUI/FeedSetupTest.java | 61 ++-- .../regression/searchUI/HomePageTest.java | 4 +- .../searchUI/MirrorSourceTargetOptionsTest.java | 2 + .../falcon/regression/searchUI/MirrorTest.java | 76 ++++- .../regression/searchUI/ProcessSetupTest.java | 65 ++-- .../regression/security/FalconClientTest.java | 7 +- .../triage/PipelineInstanceDependencyTest.java | 335 +++++++++++++++++++ 33 files changed, 1365 insertions(+), 285 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/CHANGES.txt ---------------------------------------------------------------------- diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt index 5ea229a..d285661 100644 --- a/falcon-regression/CHANGES.txt +++ b/falcon-regression/CHANGES.txt @@ -5,6 +5,9 @@ Trunk (Unreleased) INCOMPATIBLE CHANGES NEW FEATURES + FALCON-1546 Add ProcessUpdateTest, PipelineInstanceDependencyTest and other tests and test fixes + (Raghav Gautam and Paul Isaychuk via Paul Isaychuk) + FALCON-1387 Add Instance Dependency API Test(Pragya Mittal via Ajay Yadava) FALCON-1382 Add a test for feed retention to make sure that data directory is not deleted (Paul Isaychuk) http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java index b905bee..7607aa6 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java @@ -213,6 +213,15 @@ public class ProcessMerlin extends Process { return this; } + public String getProperty(String name) { + for (Property property : properties.getProperties()) { + if (property.getName().equals(name)) { + return property.getValue(); + } + } + return null; + } + @Override public String toString() { try { http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java index 83d06a2..e406cae 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/entity/AbstractEntityHelper.java @@ -38,6 +38,7 @@ import org.apache.falcon.resource.FeedInstanceResult; import org.apache.falcon.resource.InstanceDependencyResult; import org.apache.falcon.resource.InstancesResult; import org.apache.falcon.resource.InstancesSummaryResult; +import org.apache.falcon.resource.TriageResult; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.security.authentication.client.AuthenticationException; @@ -554,6 +555,15 @@ public abstract class AbstractEntityHelper { } /** + * Retrieves instance triage. + */ + public TriageResult getInstanceTriage(String entityName, String params) + throws AuthenticationException, IOException, URISyntaxException, InterruptedException { + String url = createUrl(this.hostname + URLS.INSTANCE_TRIAGE.getValue(), getEntityType(), entityName); + return (TriageResult) InstanceUtil.createAndSendRequestProcessInstance(url, params, allColo, null); + } + + /** * Lists all entities which are tagged by a given pipeline. * @param pipeline filter * @return service response http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/EntityLineageUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/EntityLineageUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/EntityLineageUtil.java index fc42cf5..2df474d 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/EntityLineageUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/EntityLineageUtil.java @@ -20,10 +20,14 @@ package org.apache.falcon.regression.core.util; import org.apache.falcon.resource.LineageGraphResult; import org.apache.log4j.Logger; +import org.joda.time.DateTime; import org.testng.Assert; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; +import java.util.List; +import java.util.Map; import java.util.Set; @@ -34,6 +38,13 @@ public final class EntityLineageUtil{ private static final Logger LOGGER = Logger.getLogger(EntityLineageUtil.class); + /** + * Enum to represent entity role in pipeline. + */ + public enum PipelineEntityType { + PROCESS, INPUT_FEED, OUTPUT_FEED + } + private EntityLineageUtil() { throw new AssertionError("Instantiating utility class..."); } @@ -61,5 +72,57 @@ public final class EntityLineageUtil{ Assert.assertEquals(expectedVerticesSet, actualVerticesSet, "Vertices dont match"); } + /** + * Produces list of expected vertices and edges in triage result. + */ + public static LineageGraphResult getExpectedResult(int bundleIndx, + Map<PipelineEntityType, List<String>> entityNamesMap, + List<Integer> inputFeedFrequencies, String entityName, + String clusterName, String startTime) { + List<String> processNames = entityNamesMap.get(PipelineEntityType.PROCESS); + List<String> inputFeedNames = entityNamesMap.get(PipelineEntityType.INPUT_FEED); + List<String> outputFeedNames = entityNamesMap.get(PipelineEntityType.OUTPUT_FEED); + List<String> vertices = new ArrayList<>(); + List<LineageGraphResult.Edge> edges = new ArrayList<>(); + final String startTimeMinus20 = TimeUtil.addMinsToTime(startTime, -20); + String vertexTemplate = "name: %s, type: %s, cluster: %s, instanceTime: %s, tags: %s"; + for (int i = 0; i <= bundleIndx; ++i) { + //add vertex of i-th bundle process + boolean isTerminalInstance = processNames.contains(entityName) && i == bundleIndx; + String tag = isTerminalInstance ? "[WAITING]" : "Output[WAITING]"; + final String processVertex = String.format(vertexTemplate, + processNames.get(i), "PROCESS", clusterName, startTime, tag); + vertices.add(processVertex); + + //add all input feed vertices & edges for i-th bundle + LineageGraphResult.Edge edge; + String feedVertex; + for (DateTime dt = new DateTime(startTime); !dt.isBefore(new DateTime(startTimeMinus20)); + dt = dt.minusMinutes(inputFeedFrequencies.get(i))) { + feedVertex = String.format(vertexTemplate, inputFeedNames.get(i), "FEED", + clusterName, TimeUtil.dateToOozieDate(dt.toDate()), "Input[MISSING]"); + edge = new LineageGraphResult.Edge(feedVertex, processVertex, "consumed by"); + vertices.add(feedVertex); + edges.add(edge); + } + //add output feed edge for i-th bundle + tag = (outputFeedNames.contains(entityName) && i == bundleIndx) ? "[MISSING]" : "Input[MISSING]"; + feedVertex = String.format(vertexTemplate, outputFeedNames.get(i), "FEED", clusterName, startTime, tag); + isTerminalInstance = i == bundleIndx && outputFeedNames.contains(entityName); + if (i < bundleIndx || isTerminalInstance) { + edge = new LineageGraphResult.Edge(processVertex, feedVertex, "produces"); + edges.add(edge); + } + //add output feed vertex only if it is terminal; it will be added as the input for next bundle otherwise + if (isTerminalInstance) { + vertices.add(feedVertex); + } + } + LineageGraphResult lineageGraphResult = new LineageGraphResult(); + lineageGraphResult.setVertices(vertices.toArray(new String[vertices.size()])); + lineageGraphResult.setEdges(edges.toArray(new LineageGraphResult.Edge[edges.size()])); + return lineageGraphResult; + } + } http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java index 10463c2..3d05ae9 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java @@ -36,6 +36,7 @@ import org.apache.falcon.resource.InstanceDependencyResult; import org.apache.falcon.resource.InstancesResult; import org.apache.falcon.resource.InstancesSummaryResult; import org.apache.falcon.resource.SchedulableEntityInstance; +import org.apache.falcon.resource.TriageResult; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.http.HttpResponse; @@ -96,8 +97,10 @@ public final class InstanceUtil { result = new InstancesSummaryResult(APIResult.Status.FAILED, responseString); }else if (url.contains("/listing/")) { result = new FeedInstanceResult(APIResult.Status.FAILED, responseString); - }else if (url.contains("/dependencies/")) { + }else if (url.contains("instance/dependencies")) { result = new InstanceDependencyResult(APIResult.Status.FAILED, responseString); + }else if (url.contains("instance/triage")) { + result = new TriageResult(APIResult.Status.FAILED, responseString); }else { result = new InstancesResult(APIResult.Status.FAILED, responseString); } @@ -126,10 +129,7 @@ public final class InstanceUtil { public Date deserialize(JsonElement json, Type t, JsonDeserializationContext c) { return new DateTime(json.getAsString()).toDate(); } - }).create().fromJson(responseString, - url.contains("/listing/") ? FeedInstanceResult.class : url.contains("/summary/") - ? InstancesSummaryResult.class : url.contains("/dependencies/") - ? InstanceDependencyResult.class : InstancesResult.class); + }).create().fromJson(responseString, getClassOfResult(url)); } catch (JsonSyntaxException e) { Assert.fail("Not a valid json:\n" + responseString); } @@ -140,6 +140,25 @@ public final class InstanceUtil { } /** + * Returns API result class matching to API request url. + */ + private static Class<? extends APIResult> getClassOfResult(String url) { + final Class<? extends APIResult> classOfResult; + if (url.contains("/listing/")) { + classOfResult = FeedInstanceResult.class; + } else if (url.contains("/summary/")) { + classOfResult = InstancesSummaryResult.class; + } else if (url.contains("instance/dependencies")) { + classOfResult = InstanceDependencyResult.class; + } else if (url.contains("instance/triage")) { + classOfResult = TriageResult.class; + } else { + classOfResult = InstancesResult.class; + } + return classOfResult; + } + + /** * Checks if API response reflects success and if it's instances match to expected status. * * @param instancesResult - kind of response from API which should contain information about @@ -717,8 +736,8 @@ public final class InstanceUtil { * @throws ParseException */ public static void assertProcessInstances(InstanceDependencyResult instancesResult, OozieClient oozieClient, - String bundleID, String time) throws OozieClientException, - JSONException, ParseException { + String bundleID, String time) + throws OozieClientException, ParseException, JSONException { List<String> inputPath = new ArrayList<>(); List<String> outputPath = new ArrayList<>(); SchedulableEntityInstance[] instances = instancesResult.getDependencies(); @@ -808,13 +827,10 @@ public final class InstanceUtil { * @param processName process name for given bundle * @param tag Input/Output * @param expectedInstances instance for given instanceTime. - * @throws JSONException * @throws ParseException - * @throws OozieClientException */ public static void assertFeedInstances(InstanceDependencyResult instancesResult, String processName, String tag, - List<String> expectedInstances) - throws OozieClientException, JSONException, ParseException { + List<String> expectedInstances) throws ParseException { List<String> actualInstances = new ArrayList<>(); SchedulableEntityInstance[] instances = instancesResult.getDependencies(); LOGGER.info("instances: " + Arrays.toString(instances)); @@ -833,7 +849,7 @@ public final class InstanceUtil { Set<String> expectedInstancesSet = new HashSet<>(expectedInstances); Set<String> actualInstancesSet = new HashSet<>(actualInstances); - Assert.assertEquals(expectedInstancesSet, actualInstancesSet, "Instances dont match"); + Assert.assertEquals(expectedInstancesSet, actualInstancesSet, "Instances don't match"); } } http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java index 9d028fa..c9f540f 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java @@ -18,6 +18,7 @@ package org.apache.falcon.regression.core.util; +import org.apache.commons.exec.CommandLine; import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; import org.apache.hadoop.security.UserGroupInformation; @@ -41,6 +42,14 @@ public final class KerberosHelper { getKeyTab(user)); } + /** + * Switches user in kerberos. + */ + public static void initUserWithKeytab(String user){ + ExecUtil.executeCommand(new CommandLine("sudo").addArgument("kinit").addArgument(getPrincipal(user)) + .addArgument("-k").addArgument("-t").addArgument(getKeyTab(user))); + } + private static String getKeyTab(String user) { return MerlinConstants.getKeytabForUser(user); } http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java index 5e2c7b2..ae96044 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java @@ -37,6 +37,10 @@ import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; import org.json.JSONException; import org.testng.Assert; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -45,6 +49,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.EnumSet; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -754,4 +759,51 @@ public final class OozieUtil { } return conf; } + + /** + * Method retrieves and parses replication coordinator action workflow definition and checks whether specific + * properties are present in list of workflow args or not. + * @param workflowDefinition workflow definition + * @param actionName action within workflow, e.g pre-processing, replication etc. + * @param propMap specific properties which are expected to be in arg list + * @return true if all keys and values are present, false otherwise + */ + public static boolean propsArePresentInWorkflow(String workflowDefinition, String actionName, + HashMap<String, String> propMap) { + //get action definition + Document definition = Util.convertStringToDocument(workflowDefinition); + Assert.assertNotNull(definition, "Workflow definition shouldn't be null."); + NodeList actions = definition.getElementsByTagName("action"); + Element action = null; + for (int i = 0; i < actions.getLength(); i++) { + Node node = actions.item(i); + if (node.getNodeType() == Node.ELEMENT_NODE) { + action = (Element) node; + if (action.getAttribute("name").equals(actionName)) { + break; + } + action = null; + } + } + Assert.assertNotNull(action, actionName + " action not found."); + + //retrieve and checks whether properties are present in workflow args + Element javaElement = (Element) action.getElementsByTagName("java").item(0); + NodeList args = javaElement.getElementsByTagName("arg"); + int counter = 0; + String key = null; + for (int i = 0; i < args.getLength(); i++) { + Node node = args.item(i); + if (node.getNodeType() == Node.ELEMENT_NODE) { + String argKey = node.getTextContent().replace("-", ""); + if (key != null && propMap.get(key).equals(argKey)) { + counter++; + key = null; + } else if (key == null && propMap.containsKey(argKey)) { + key = argKey; + } + } + } + return counter == propMap.size(); + } } http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java index 83547e7..ccd083b 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/Util.java @@ -46,6 +46,7 @@ import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; import org.testng.Assert; +import org.w3c.dom.Document; import org.xml.sax.InputSource; import org.xml.sax.SAXException; @@ -54,6 +55,8 @@ import javax.jms.MapMessage; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.transform.OutputKeys; import javax.xml.transform.Source; import javax.xml.transform.Transformer; @@ -391,6 +394,7 @@ public final class Util { INSTANCE_RERUN("/api/instance/rerun"), INSTANCE_SUMMARY("/api/instance/summary"), INSTANCE_PARAMS("/api/instance/params"), + INSTANCE_TRIAGE("/api/instance/triage"), INSTANCE_LIST("/api/instance/list"), INSTANCE_LISTING("/api/instance/listing"), INSTANCE_LOGS("/api/instance/logs"), @@ -563,6 +567,24 @@ public final class Util { } /** + * Converts string to xml document. + * @param xmlStr string representation + * @return document representation. + */ + public static Document convertStringToDocument(String xmlStr) { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder; + try { + builder = factory.newDocumentBuilder(); + Document doc = builder.parse(new InputSource(new StringReader(xmlStr))); + return doc; + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + + /** * Sends api requests. * @param url target url * @param method request method http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/AbstractSearchPage.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/AbstractSearchPage.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/AbstractSearchPage.java index d956549..ab73092 100644 --- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/AbstractSearchPage.java +++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/AbstractSearchPage.java @@ -20,14 +20,15 @@ package org.apache.falcon.regression.ui.search; import com.google.common.util.concurrent.SimpleTimeLimiter; import com.google.common.util.concurrent.TimeLimiter; -import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.commons.lang3.tuple.MutablePair; import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; import org.apache.falcon.regression.core.util.TimeUtil; import org.apache.falcon.regression.ui.pages.Page; import org.apache.log4j.Logger; import org.openqa.selenium.By; import org.openqa.selenium.JavascriptExecutor; +import org.openqa.selenium.NoSuchElementException; import org.openqa.selenium.TimeoutException; import org.openqa.selenium.WebDriver; import org.openqa.selenium.WebElement; @@ -50,11 +51,14 @@ public abstract class AbstractSearchPage extends Page { public static final String UI_URL = MerlinConstants.PRISM_URL; private static final Logger LOGGER = Logger.getLogger(AbstractSearchPage.class); public static final int PAGELOAD_TIMEOUT_THRESHOLD = 10; + public static final int ALERT_LIFETIME = 3000; public AbstractSearchPage(WebDriver driver) { super(driver); waitForAngularToFinish(); + LOGGER.info("Going to initialize Page Header."); pageHeader = PageFactory.initElements(driver, PageHeader.class); + LOGGER.info("Initialization done."); } private PageHeader pageHeader; @@ -163,13 +167,54 @@ public abstract class AbstractSearchPage extends Page { public String getActiveAlertText() { if (waitForAlert()) { waitForAngularToFinish(); - return driver.findElement(By.xpath("//div[@class='messages notifs']/div[last()]")).getText(); + String script = "return $('div.messages.notifs > div:last-child').text();"; + String message = (String)((JavascriptExecutor)driver).executeScript(script); + return message.trim(); } else { return null; } } /** + * Wait for active alert. Check it's lifetime (the period when alert is displayed). + */ + public void validateAlertLifetime() { + final WebElement alertsBlock = driver.findElement(By.xpath("//div[@class='messages notifs']")); + try { + final MutablePair<Long, Long> pair = new MutablePair<>(Long.MAX_VALUE, Long.MAX_VALUE); + // wait 5 seconds for alert to start blinking and record time of first blink + new WebDriverWait(driver, 5, 100).until(new ExpectedCondition<Boolean>() { + @Nullable + @Override + public Boolean apply(WebDriver webDriver) { + String style = alertsBlock.getAttribute("style"); + if ((style.contains("opacity") && !style.contains("opacity: 1;")) + || style.contains("display: block;")) { + pair.setLeft(System.currentTimeMillis()); + return true; + } + return false; + } + }); + // wait 5 seconds for alert to stop blinking and record time of stoppage + for (int i = 0; i < ALERT_LIFETIME + 3000; i += 100) { + String style = alertsBlock.getAttribute("style"); + if (style.contains("display: none;")) { + pair.setRight(Math.min(System.currentTimeMillis(), pair.getRight())); + } else { + pair.setRight(Long.MAX_VALUE); + } + TimeUtil.sleepSeconds(0.1); + } + long diff = pair.getRight() - pair.getLeft(); + LOGGER.info(String.format("Alert was live %d millis.", pair.getRight() - pair.getLeft())); + Assert.assertTrue(ALERT_LIFETIME <= diff, "Alert was present for too short period of time"); + } catch (TimeoutException e) { + Assert.fail("Alert didn't appear in 5 seconds."); + } + } + + /** * Wait for active alert. * @return true is alert is present */ @@ -181,12 +226,8 @@ public abstract class AbstractSearchPage extends Page { @Override public Boolean apply(WebDriver webDriver) { String style = alertsBlock.getAttribute("style"); - if (style.contains("opacity") && !style.contains("opacity: 1;")) { - String alert = alertsBlock.findElement(By.xpath("./div[last()]")).getText(); - return StringUtils.isNotEmpty(alert); - } else { - return false; - } + return (style.contains("opacity") && !style.contains("opacity: 1;")) + || style.contains("display: block;"); } }); return true; @@ -196,6 +237,18 @@ public abstract class AbstractSearchPage extends Page { } /** + * Performs simple check of element presence. + */ + public WebElement getElementOrNull(String xpath) { + try { + return driver.findElement(By.xpath(xpath)); + } catch (NoSuchElementException ignored) { + return null; + } + } + + + /** * Method imitates click on check box. If click is not performed method retries the click. * @param expectedState whether check box is expected to be enabled or not after click. */ http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ClusterWizardPage.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ClusterWizardPage.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ClusterWizardPage.java index 0fbfc38..bcada4a 100644 --- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ClusterWizardPage.java +++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ClusterWizardPage.java @@ -38,7 +38,7 @@ import org.testng.Assert; import java.util.List; /** Page object of the Cluster creation page. */ -public class ClusterWizardPage extends AbstractSearchPage { +public class ClusterWizardPage extends EntityWizardPage { private static final Logger LOGGER = Logger.getLogger(ClusterWizardPage.class); @FindBys({ @FindBy(className = "mainUIView"), @@ -53,12 +53,8 @@ public class ClusterWizardPage extends AbstractSearchPage { private WebElement previous; @FindBy(xpath = "//a[contains(text(), 'Cancel')]") private WebElement cancel; - @FindBy(id = "cluster.editXML") - private WebElement editXML; @FindBy(xpath = "//div[contains(@class, 'clusterSummaryRow')][h4]") private WebElement summaryBox; - @FindBy(xpath = "//div[contains(@class, 'xmlPreviewContainer')]//textarea") - private WebElement xmlPreview; public ClusterWizardPage(WebDriver driver) { super(driver); @@ -273,21 +269,6 @@ public class ClusterWizardPage extends AbstractSearchPage { return false; } - public ClusterMerlin getXmlPreview() { - //preview block fetches changes slower then they appear on the form - waitForAngularToFinish(); - String previewData = xmlPreview.getAttribute("value"); - return new ClusterMerlin(previewData); - } - - public void setClusterXml(String clusterXml) { - clickEditXml(true); - xmlPreview.clear(); - xmlPreview.sendKeys(clusterXml); - waitForAngularToFinish(); - clickEditXml(false); - } - /** * Retrieves hte value of the summary box and parses it to cluster properties. * @param draft empty cluster to contain all properties. @@ -402,16 +383,6 @@ public class ClusterWizardPage extends AbstractSearchPage { } /** - * Clicks on editXml button. - */ - public void clickEditXml(boolean shouldBeEnabled) { - editXML.click(); - String disabled = xmlPreview.getAttribute("disabled"); - Assert.assertEquals(disabled == null, shouldBeEnabled, - "Xml preview should be " + (shouldBeEnabled ? "enabled" : "disabled")); - } - - /** * Click on next button which is the same as finish step 1. */ public void clickNext() { @@ -486,4 +457,13 @@ public class ClusterWizardPage extends AbstractSearchPage { } } + @Override + public WebElement getEditXMLButton() { + return driver.findElement(By.id("cluster.editXML")); + } + + @Override + public ClusterMerlin getEntityFromXMLPreview() { + return new ClusterMerlin(getXMLPreview()); + } } http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/EntityWizardPage.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/EntityWizardPage.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/EntityWizardPage.java new file mode 100644 index 0000000..72c03cf --- /dev/null +++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/EntityWizardPage.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.regression.ui.search; + +import org.apache.falcon.entity.v0.Entity; +import org.openqa.selenium.WebDriver; +import org.openqa.selenium.WebElement; +import org.openqa.selenium.support.FindBy; +import org.testng.Assert; + +/** + * https://issues.apache.org/jira/browse/FALCON-1546. + * Parent class for cluster, feed and process wizard pages. + */ +public abstract class EntityWizardPage extends AbstractSearchPage { + @FindBy(xpath = "//i[contains(@class, 'pointer')]") + protected WebElement xmlPreviewPointer; + protected WebElement xmlPreview = null; + + public EntityWizardPage(WebDriver driver) { + super(driver); + } + + /** + * Expand/collapse xml preview. + * @param shouldBeExpanded should preview be expanded or collapsed. + */ + public void clickXMLPreview(boolean shouldBeExpanded) { + if (isXmlPreviewExpanded() != shouldBeExpanded) { + xmlPreviewPointer.click(); + } + Assert.assertEquals(isXmlPreviewExpanded(), shouldBeExpanded, + "Xml preview should be " + (shouldBeExpanded ? " expanded." : " collapsed.")); + } + + /** + * @return true if xml preview exists and is displayed, false otherwise. + */ + public boolean isXmlPreviewExpanded() { + xmlPreview = getElementOrNull("//textarea[@ng-model='prettyXml']"); + return xmlPreview != null && xmlPreview.isDisplayed(); + } + + public String getXMLPreview() { + //preview block fetches changes slower then they appear on the form + waitForAngularToFinish(); + clickXMLPreview(true); + return xmlPreview.getAttribute("value"); + } + + public abstract Entity getEntityFromXMLPreview(); + + /** + * Pushes xml into xml preview block. + * @param xml entity definition + */ + public void setXmlPreview(String xml) { + clickEditXml(true); + xmlPreview.clear(); + xmlPreview.sendKeys(xml); + waitForAngularToFinish(); + clickEditXml(false); + } + + /** + * Clicks on editXml button. + */ + public void clickEditXml(boolean shouldBeEnabled) { + waitForAngularToFinish(); + clickXMLPreview(true); + getEditXMLButton().click(); + String disabled = xmlPreview.getAttribute("disabled"); + Assert.assertEquals(disabled == null, shouldBeEnabled, + "Xml preview should be " + (shouldBeEnabled ? "enabled" : "disabled")); + } + + public abstract WebElement getEditXMLButton(); +} http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/FeedWizardPage.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/FeedWizardPage.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/FeedWizardPage.java index f3a107c..3dfab38 100644 --- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/FeedWizardPage.java +++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/FeedWizardPage.java @@ -34,7 +34,7 @@ import java.util.Date; import java.util.List; /** Page object of the Feed creation page. */ -public class FeedWizardPage extends AbstractSearchPage { +public class FeedWizardPage extends EntityWizardPage { private static final Logger LOGGER = Logger.getLogger(FeedWizardPage.class); @@ -83,17 +83,9 @@ public class FeedWizardPage extends AbstractSearchPage { }) private WebElement saveFeedButton; - @FindBys({ - @FindBy(id = "feed.editXML") - }) - private WebElement editXmlButton; - @FindBy(xpath = "//a[contains(.,'Cancel')]") private WebElement cancelButton; - @FindBy(xpath = "//textarea[@ng-model='prettyXml']") - private WebElement feedXml; - public FeedWizardPage(WebDriver driver) { super(driver); } @@ -327,11 +319,6 @@ public class FeedWizardPage extends AbstractSearchPage { cancelButton.click(); } - public void clickEditXml(){ - waitForAngularToFinish(); - editXmlButton.click(); - } - public void clickCatalogStorageButton(){ catalogStorageButton.click(); waitForAngularToFinish(); @@ -652,14 +639,14 @@ public class FeedWizardPage extends AbstractSearchPage { waitForAlert(); } - public FeedMerlin getFeedMerlinFromFeedXml() throws Exception{ - waitForAngularToFinish(); - return FeedMerlin.fromString(feedXml.getAttribute("value")); + @Override + public FeedMerlin getEntityFromXMLPreview() { + return FeedMerlin.fromString(getXMLPreview()); } - public void setFeedXml(String xml) throws Exception{ - feedXml.clear(); - feedXml.sendKeys(xml); + @Override + public WebElement getEditXMLButton() { + return driver.findElement(By.id("feed.editXML")); } } http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/LoginPage.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/LoginPage.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/LoginPage.java index 3193d21..5b261fb 100644 --- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/LoginPage.java +++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/LoginPage.java @@ -46,6 +46,7 @@ public class LoginPage extends AbstractSearchPage { public static LoginPage open(WebDriver driver) { driver.get(UI_URL); + LOGGER.info("Opened a URL: " + UI_URL); return PageFactory.initElements(driver, LoginPage.class); } http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/MirrorWizardPage.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/MirrorWizardPage.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/MirrorWizardPage.java index 6dfa1ca..f990c92 100644 --- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/MirrorWizardPage.java +++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/MirrorWizardPage.java @@ -278,7 +278,12 @@ public class MirrorWizardPage extends AbstractSearchPage { return new ClusterBlock("Target"); } - public void applyRecipe(RecipeMerlin recipe) { + /** + * Populates hive dr UI with parameters from recipe. + * @param recipe recipe + * @param overwriteDefaults should it overwrite HiveDR default values automatically picked up by UI + */ + public void applyRecipe(RecipeMerlin recipe, boolean overwriteDefaults) { final ClusterMerlin srcCluster = recipe.getSrcCluster(); final ClusterMerlin tgtCluster = recipe.getTgtCluster(); setName(recipe.getName()); @@ -303,8 +308,10 @@ public class MirrorWizardPage extends AbstractSearchPage { setHiveReplicationMaxMaps(recipe.getReplicationMaxMaps()); setMaxEvents(recipe.getMaxEvents()); setHiveMaxBandwidth(recipe.getMapBandwidth()); - setSourceInfo(recipe.getSrcCluster()); - setTargetInfo(recipe.getTgtCluster()); + if (overwriteDefaults) { + setSourceInfo(recipe.getSrcCluster()); + setTargetInfo(recipe.getTgtCluster()); + } break; default: break; http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/PageHeader.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/PageHeader.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/PageHeader.java index 2a75b20..7f87091 100644 --- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/PageHeader.java +++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/PageHeader.java @@ -159,8 +159,8 @@ public class PageHeader { //checking if logged-in username is displayed if (!MerlinConstants.IS_SECURE) { UIAssert.assertDisplayed(getLogoutButton(), "Logout button"); + AssertUtil.assertNotEmpty(getLoggedInUser(), "Expecting logged-in username."); } - AssertUtil.assertNotEmpty(getLoggedInUser(), "Expecting logged-in username."); //create button navigation doCreateCluster(); @@ -249,7 +249,33 @@ public class PageHeader { } private WebElement getLogoutButton() { - return loginHeaderBox.findElements(By.tagName("button")).get(1); + return loginHeaderBox.findElements(By.xpath("button[@ng-click='logOut()']")).get(0); + } + + private WebElement getNotificationButton() { + return loginHeaderBox.findElements(By.xpath("button[@ng-click='notify()']")).get(0); + } + + /** + * Validates number of notifications contained by notification bar and last notification message. + */ + public void validateNotificationCountAndCheckLast(int count, String message) { + WebElement notificationButton = getNotificationButton(); + notificationButton.click(); + waitForAngularToFinish(); + + // Test notifications dropdown visibility + WebElement notificationDropdown = notificationButton.findElement(By.className("messages")); + Assert.assertTrue(notificationDropdown.getAttribute("style").contains("display: block;"), + "Notifications are not visible."); + + // Test validity of number of notifications + List<WebElement> notifications = notificationDropdown.findElements(By.xpath("div")); + Assert.assertEquals(notifications.size() - 1, count, "Invalid notification count."); + + // Test validity of last notification + String lastNotification = notifications.get(0).getText(); + Assert.assertTrue(lastNotification.contains(message), "Invalid last notification text."); } public LoginPage doLogout() { http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ProcessWizardPage.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ProcessWizardPage.java b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ProcessWizardPage.java index 706328f..5dcd700 100644 --- a/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ProcessWizardPage.java +++ b/falcon-regression/merlin/src/main/java/org/apache/falcon/regression/ui/search/ProcessWizardPage.java @@ -32,8 +32,8 @@ import org.apache.falcon.entity.v0.process.Output; import org.apache.falcon.entity.v0.process.Outputs; import org.apache.falcon.entity.v0.process.PolicyType; import org.apache.falcon.entity.v0.process.Retry; -import org.apache.falcon.entity.v0.process.Workflow; import org.apache.falcon.entity.v0.process.Validity; +import org.apache.falcon.entity.v0.process.Workflow; import org.apache.falcon.regression.Entities.ProcessMerlin; import org.apache.falcon.regression.core.util.UIAssert; import org.apache.log4j.Logger; @@ -52,7 +52,7 @@ import java.util.List; import java.util.TimeZone; /** Page object of the Process creation page. */ -public class ProcessWizardPage extends AbstractSearchPage { +public class ProcessWizardPage extends EntityWizardPage { private static final Logger LOGGER = Logger.getLogger(ProcessWizardPage.class); @@ -62,9 +62,6 @@ public class ProcessWizardPage extends AbstractSearchPage { }) private WebElement processBox; - @FindBy(xpath = "//textarea[@ng-model='prettyXml']") - private WebElement processXml; - @FindBy(xpath = "//form[@name='processForm']/div[1]") private WebElement summaryBox; @@ -82,15 +79,10 @@ public class ProcessWizardPage extends AbstractSearchPage { }) private WebElement previousButton; - @FindBys({ - @FindBy(id = "editXmlButton") - }) - private WebElement editXmlButton; - @FindBy(xpath = "//a[contains(.,'Cancel')]") private WebElement cancelButton; - @FindBy(xpath = "//div[contains(@class,'formBoxContainer')]") + @FindBy(xpath = "//fieldset[@id='fieldWrapper']") private WebElement formBox; public ProcessWizardPage(WebDriver driver) { @@ -130,11 +122,6 @@ public class ProcessWizardPage extends AbstractSearchPage { cancelButton.click(); } - public void clickEditXml(){ - waitForAngularToFinish(); - editXmlButton.click(); - } - /*----- Step1 General info ----*/ private WebElement getName() { @@ -828,20 +815,14 @@ public class ProcessWizardPage extends AbstractSearchPage { waitForAlert(); } - /** - * Creates ProcessMerlin object from xml preview string. - */ - public ProcessMerlin getProcessMerlinFromProcessXml() throws Exception{ - waitForAngularToFinish(); - return new ProcessMerlin(processXml.getAttribute("value")); + @Override + public ProcessMerlin getEntityFromXMLPreview() { + return new ProcessMerlin(getXMLPreview()); } - /** - * Pushes xml string to xml preview. - */ - public void setProcessXml(String xml) throws Exception{ - processXml.clear(); - processXml.sendKeys(xml); + @Override + public WebElement getEditXMLButton() { + return driver.findElement(By.id("editXmlButton")); } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java index 0662562..728b797 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ExternalFSTest.java @@ -182,9 +182,9 @@ public class ExternalFSTest extends BaseTestClass{ Path dstPath = new Path(endpoint + testWasbTargetDir + '/' + timePattern); //check if coordinator exists + TimeUtil.sleepSeconds(10); InstanceUtil.waitTillInstancesAreCreated(clusterOC, feed.toString(), 0); Assert.assertEquals(OozieUtil.checkIfFeedCoordExist(clusterOC, feed.getName(), "REPLICATION"), 1); - TimeUtil.sleepSeconds(10); //replication should start, wait while it ends InstanceUtil.waitTillInstanceReachState(clusterOC, Util.readEntityName(feed.toString()), 1, http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java index 9ac9f24..6728edf 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/FeedReplicationTest.java @@ -18,11 +18,11 @@ package org.apache.falcon.regression; -import org.apache.falcon.regression.Entities.FeedMerlin; -import org.apache.falcon.regression.core.bundle.Bundle; import org.apache.falcon.entity.v0.EntityType; import org.apache.falcon.entity.v0.feed.ActionType; import org.apache.falcon.entity.v0.feed.ClusterType; +import org.apache.falcon.regression.Entities.FeedMerlin; +import org.apache.falcon.regression.core.bundle.Bundle; import org.apache.falcon.regression.core.helpers.ColoHelper; import org.apache.falcon.regression.core.util.AssertUtil; import org.apache.falcon.regression.core.util.BundleUtil; @@ -36,9 +36,11 @@ import org.apache.falcon.regression.testHelper.BaseTestClass; import org.apache.falcon.resource.InstancesResult; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.security.authentication.client.AuthenticationException; import org.apache.log4j.Logger; import org.apache.oozie.client.CoordinatorAction; import org.apache.oozie.client.OozieClient; +import org.apache.oozie.client.OozieClientException; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; @@ -51,7 +53,10 @@ import org.testng.annotations.Test; import javax.xml.bind.JAXBException; import java.io.IOException; +import java.net.URISyntaxException; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * feed replication test. @@ -383,6 +388,106 @@ public class FeedReplicationTest extends BaseTestClass { } /** + * Test for https://issues.apache.org/jira/browse/FALCON-668. + * Check that new DistCp options are allowed. + */ + @Test + public void testNewDistCpOptions() + throws URISyntaxException, AuthenticationException, InterruptedException, IOException, JAXBException, + OozieClientException { + Bundle.submitCluster(bundles[0], bundles[1]); + String startTime = TimeUtil.getTimeWrtSystemTime(0); + String endTime = TimeUtil.addMinsToTime(startTime, 5); + LOGGER.info("Time range between : " + startTime + " and " + endTime); + //configure feed + String feedName = Util.readEntityName(bundles[0].getDataSets().get(0)); + FeedMerlin feedElement = bundles[0].getFeedElement(feedName); + bundles[0].writeFeedElement(feedElement, feedName); + FeedMerlin feed = new FeedMerlin(bundles[0].getDataSets().get(0)); + feed.setFilePath(feedDataLocation); + //erase all clusters from feed definition + feed.clearFeedClusters(); + //set cluster1 as source + feed.addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[0].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity(startTime, endTime) + .withClusterType(ClusterType.SOURCE) + .build()); + //set cluster2 as target + feed.addFeedCluster( + new FeedMerlin.FeedClusterBuilder(Util.readEntityName(bundles[1].getClusters().get(0))) + .withRetention("days(1000000)", ActionType.DELETE) + .withValidity(startTime, endTime) + .withClusterType(ClusterType.TARGET) + .withDataLocation(targetDataLocation) + .build()); + + //add custom properties to feed + HashMap<String, String> propMap = new HashMap<>(); + propMap.put("overwrite", "true"); + propMap.put("ignoreErrors", "false"); + propMap.put("skipChecksum", "false"); + propMap.put("removeDeletedFiles", "true"); + propMap.put("preserveBlockSize", "true"); + propMap.put("preserveReplicationNumber", "true"); + propMap.put("preservePermission", "true"); + for (Map.Entry<String, String> entry : propMap.entrySet()) { + feed.withProperty(entry.getKey(), entry.getValue()); + } + //add custom property which shouldn't be passed to workflow + HashMap<String, String> unsupportedPropMap = new HashMap<>(); + unsupportedPropMap.put("myCustomProperty", "true"); + feed.withProperty("myCustomProperty", "true"); + + //upload necessary data to source + DateTime date = new DateTime(startTime, DateTimeZone.UTC); + DateTimeFormatter fmt = DateTimeFormat.forPattern("yyyy'/'MM'/'dd'/'HH'/'mm'"); + String timePattern = fmt.print(date); + String sourceLocation = sourcePath + "/" + timePattern + "/"; + HadoopUtil.recreateDir(cluster1FS, sourceLocation); + HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile.xml")); + HadoopUtil.copyDataToFolder(cluster1FS, sourceLocation, OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile1.txt")); + + //copy 2 files to target to check if they will be deleted because of removeDeletedFiles property + String targetLocation = targetPath + "/" + timePattern + "/"; + cluster2FS.copyFromLocalFile(new Path(OSUtil.concat(OSUtil.NORMAL_INPUT, "dataFile3.txt")), + new Path(targetLocation + "dataFile3.txt")); + + //submit and schedule feed + LOGGER.info("Feed : " + Util.prettyPrintXml(feed.toString())); + AssertUtil.assertSucceeded(prism.getFeedHelper().submitAndSchedule(feed.toString())); + + //check while instance is got created + InstanceUtil.waitTillInstancesAreCreated(cluster2OC, feed.toString(), 0); + + //check if coordinator exists and replication starts + Assert.assertEquals(OozieUtil.checkIfFeedCoordExist(cluster2OC, feed.getName(), "REPLICATION"), 1); + InstanceUtil.waitTillInstanceReachState(cluster2OC, feed.getName(), 1, + CoordinatorAction.Status.RUNNING, EntityType.FEED); + + //check that properties were passed to workflow definition + String bundleId = OozieUtil.getLatestBundleID(cluster2OC, feedName, EntityType.FEED); + String coordId = OozieUtil.getReplicationCoordID(bundleId, cluster2.getFeedHelper()).get(0); + CoordinatorAction coordinatorAction = cluster2OC.getCoordJobInfo(coordId).getActions().get(0); + String wfDefinition = cluster2OC.getJobDefinition(coordinatorAction.getExternalId()); + LOGGER.info(String.format("Definition of coordinator job action %s : \n %s \n", + coordinatorAction.getExternalId(), Util.prettyPrintXml(wfDefinition))); + Assert.assertTrue(OozieUtil.propsArePresentInWorkflow(wfDefinition, "replication", propMap), + "New distCp supported properties should be passed to replication args list."); + Assert.assertFalse(OozieUtil.propsArePresentInWorkflow(wfDefinition, "replication", unsupportedPropMap), + "Unsupported properties shouldn't be passed to replication args list."); + + //check that replication succeeds + InstanceUtil.waitTillInstanceReachState(cluster2OC, feed.getName(), 1, + CoordinatorAction.Status.SUCCEEDED, EntityType.FEED); + + List<Path> finalFiles = HadoopUtil.getAllFilesRecursivelyHDFS(cluster2FS, new Path(targetPath)); + Assert.assertEquals(finalFiles.size(), 2, "Only replicated files should be present on target " + + "because of 'removeDeletedFiles' distCp property."); + } + + /** * Test demonstrates failure pf replication of stored data from one source cluster to one target cluster. * When replication job fails test checks if failed logs are present in staging directory or not. */ http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java index 7f1e445..6493133 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java @@ -434,6 +434,45 @@ public class ProcessInstanceStatusTest extends BaseTestClass { InstanceUtil.validateFailedInstances(r, 3); } + /** + * Check that default end time param value is now. + */ + @Test + public void testDefaultEndTimeParam() + throws OozieClientException, IOException, InterruptedException, AuthenticationException, URISyntaxException, + JAXBException { + //set validity to have 12 instances + String start = TimeUtil.getTimeWrtSystemTime(-60); + String end = TimeUtil.getTimeWrtSystemTime(0); + bundles[0].setProcessValidity(start, end); + bundles[0].setProcessPeriodicity(5, TimeUnit.minutes); + bundles[0].setProcessConcurrency(3); + bundles[0].submitFeedsScheduleProcess(prism); + InstanceUtil.waitTillInstancesAreCreated(clusterOC, bundles[0].getProcessData(), 0); + //make first 3 instances running + OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 0); + OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 1); + OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 2); + InstanceUtil.waitTillInstanceReachState(clusterOC, processName, 3, Status.RUNNING, + EntityType.PROCESS); + //check instances status with end, expected first 10 instances + InstancesResult r = prism.getProcessHelper().getProcessInstanceStatus(processName, + "?start=" + start + "&end=" + TimeUtil.addMinsToTime(end, -11)); + InstanceUtil.validateResponse(r, 10, 3, 0, 7, 0); + //request the same but without end, expected to have the latest 10 instances + r = prism.getProcessHelper().getProcessInstanceStatus(processName, + "?start=" + start); + InstanceUtil.validateResponse(r, 10, 1, 0, 9, 0); + //the same with numResults which includes/excludes all running instances + r = prism.getProcessHelper().getProcessInstanceStatus(processName, + "?start=" + start + "&end=" + TimeUtil.addMinsToTime(end, -16) + "&numResults=9"); + InstanceUtil.validateResponse(r, 9, 3, 0, 6, 0); + //expected end is set to now, thus getting last 9 instances + r = prism.getProcessHelper().getProcessInstanceStatus(processName, + "?start=" + start + "&numResults=9"); + InstanceUtil.validateResponse(r, 9, 0, 0, 9, 0); + } + /* * Function to match the workflows obtained from instance status and oozie. */ http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessUpdateTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessUpdateTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessUpdateTest.java new file mode 100644 index 0000000..efbb503 --- /dev/null +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessUpdateTest.java @@ -0,0 +1,112 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.falcon.regression; + +import org.apache.falcon.entity.v0.EntityType; +import org.apache.falcon.entity.v0.Frequency; +import org.apache.falcon.entity.v0.process.LateInput; +import org.apache.falcon.entity.v0.process.LateProcess; +import org.apache.falcon.entity.v0.process.PolicyType; +import org.apache.falcon.regression.Entities.ProcessMerlin; +import org.apache.falcon.regression.core.bundle.Bundle; +import org.apache.falcon.regression.core.helpers.ColoHelper; +import org.apache.falcon.regression.core.util.AssertUtil; +import org.apache.falcon.regression.core.util.BundleUtil; +import org.apache.falcon.regression.core.util.InstanceUtil; +import org.apache.falcon.regression.core.util.OSUtil; +import org.apache.falcon.regression.core.util.OozieUtil; +import org.apache.falcon.regression.core.util.TimeUtil; +import org.apache.falcon.regression.core.util.Util; +import org.apache.falcon.regression.testHelper.BaseTestClass; +import org.apache.log4j.Logger; +import org.apache.oozie.client.OozieClient; +import org.testng.Assert; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +/** + * Tests related to update feature. + */ +@Test(groups = "embedded") +public class ProcessUpdateTest extends BaseTestClass { + + private ColoHelper cluster = servers.get(0); + private OozieClient clusterOC = serverOC.get(0); + private String baseTestHDFSDir = cleanAndGetTestDir(); + private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator"; + private String feedInputPath = baseTestHDFSDir + "/input" + MINUTE_DATE_PATTERN; + private String feedOutputPath = baseTestHDFSDir + "/output-data" + MINUTE_DATE_PATTERN; + private static final Logger LOGGER = Logger.getLogger(ProcessUpdateTest.class); + + @BeforeClass(alwaysRun = true) + public void uploadWorkflow() throws Exception { + uploadDirToClusters(aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE); + } + + @BeforeMethod(alwaysRun = true) + public void setUp() throws Exception { + Bundle bundle = BundleUtil.readELBundle(); + bundles[0] = new Bundle(bundle, servers.get(0)); + bundles[0].generateUniqueBundle(this); + bundles[0].setProcessWorkflow(aggregateWorkflowDir); + bundles[0].setInputFeedDataPath(feedInputPath); + bundles[0].setOutputFeedLocationData(feedOutputPath); + } + + /** + * Test for https://issues.apache.org/jira/browse/FALCON-99. + * Scenario: schedule a process which doesn't have late data handling and then update it to have it. + * Check that new coordinator was created. + */ + @Test + public void updateProcessWithLateData() throws Exception { + String start = TimeUtil.getTimeWrtSystemTime(-60); + String end = TimeUtil.getTimeWrtSystemTime(10); + bundles[0].submitAndScheduleAllFeeds(); + ProcessMerlin process = bundles[0].getProcessObject(); + process.setValidity(start, end); + process.setLateProcess(null); + cluster.getProcessHelper().submitAndSchedule(process.toString()); + InstanceUtil.waitTillInstancesAreCreated(clusterOC, process.toString(), 0); + String bundleId = OozieUtil.getLatestBundleID(clusterOC, process.getName(), EntityType.PROCESS); + + //update process to have late data handling + LateProcess lateProcess = new LateProcess(); + lateProcess.setDelay(new Frequency("hours(1)")); + lateProcess.setPolicy(PolicyType.EXP_BACKOFF); + LateInput lateInput = new LateInput(); + lateInput.setInput("inputData"); + lateInput.setWorkflowPath(aggregateWorkflowDir); + lateProcess.getLateInputs().add(lateInput); + process.setLateProcess(lateProcess); + LOGGER.info("Updated process xml: " + Util.prettyPrintXml(process.toString())); + AssertUtil.assertSucceeded(cluster.getProcessHelper().update(process.toString(), process.toString())); + + //check that new coordinator was created + String newBundleId = OozieUtil.getLatestBundleID(clusterOC, process.getName(), EntityType.PROCESS); + Assert.assertNotEquals(bundleId, newBundleId, "New Bundle should be created."); + } + + @AfterMethod(alwaysRun = true) + public void tearDown() { + removeTestClassEntities(); + } + +} http://git-wip-us.apache.org/repos/asf/falcon/blob/9e6d5a6c/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java index 9ea8471..ca6ee88 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TestngListener.java @@ -77,7 +77,7 @@ public class TestngListener implements ITestListener, IExecutionListener { LOGGER.info("Dumping of falcon store failed: " + e); } LOGGER.info( - String.format("Testing going to end for: %s.%s(%s) %s", result.getTestClass().getName(), + String.format("Testing going to end for: %s.%s(%s) ----- Status: %s", result.getTestClass().getName(), result.getName(), Arrays.toString(result.getParameters()), outcome)); NDC.pop(); LOGGER.info(hr);
