Repository: falcon Updated Branches: refs/heads/master 4656f692a -> e9e0c7b7c
FALCON-1699 Test fixes for RetentionTest, LineageApiTest, TouchAPIPrismAndServerTest, FeedReplicationTest and few fortifications. Contributed by Paul Isaychuk. Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/e9e0c7b7 Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/e9e0c7b7 Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/e9e0c7b7 Branch: refs/heads/master Commit: e9e0c7b7c0aed12e36c61f2cf8b4e67458fdfd06 Parents: 4656f69 Author: Pragya <[email protected]> Authored: Mon Jan 11 10:30:06 2016 +0000 Committer: Pragya <[email protected]> Committed: Mon Jan 11 10:30:06 2016 +0000 ---------------------------------------------------------------------- falcon-regression/CHANGES.txt | 2 + .../regression/core/response/lineage/Edge.java | 3 ++ .../regression/core/util/KerberosHelper.java | 9 ---- .../falcon/regression/core/util/OozieUtil.java | 52 +++++++++++++------- .../org/apache/falcon/request/BaseRequest.java | 3 ++ .../regression/TouchAPIPrismAndServerTest.java | 1 + .../regression/hcat/HCatRetentionTest.java | 9 +++- .../lineage/ListFeedInstancesTest.java | 12 ++--- .../falcon/regression/prism/RetentionTest.java | 6 ++- .../regression/security/FalconClientTest.java | 7 +-- 10 files changed, 63 insertions(+), 41 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/CHANGES.txt ---------------------------------------------------------------------- diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt index a4ae687..f2df91a 100644 --- a/falcon-regression/CHANGES.txt +++ b/falcon-regression/CHANGES.txt @@ -5,6 +5,8 @@ Trunk (Unreleased) INCOMPATIBLE CHANGES NEW FEATURES + FALCON-1699 Test fixes for RetentionTest, LineageApiTest, TouchAPIPrismAndServerTest, FeedReplicationTest and few fortifications(Paul Isaychuk via Pragya Mittal) + FALCON-1698 New tests for ProcessSetupTest, ClusterSetupTest, UI test fixes(Paul Isaychuk via Ajay Yadava) FALCON-1700 Add new test cases to HiveDRTest(Paul Isaychuk & Murali Ramasami via Ajay Yadava) http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/lineage/Edge.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/lineage/Edge.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/lineage/Edge.java index c1a7eb8..f1c408e 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/lineage/Edge.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/response/lineage/Edge.java @@ -58,6 +58,9 @@ public class Edge extends GraphEntity { @SerializedName("ninth")NINTH, @SerializedName("tenth")TENTH, @SerializedName("value")VALUE, + @SerializedName("_falcon_mirroring_type")MIRRORING_TYPE, + @SerializedName("specific")SPECIFIC, + @SerializedName("myTag1")MY_TAG } @SerializedName("_id") private String id; http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java index c9f540f..9d028fa 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/KerberosHelper.java @@ -18,7 +18,6 @@ package org.apache.falcon.regression.core.util; -import org.apache.commons.exec.CommandLine; import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; import org.apache.hadoop.security.UserGroupInformation; @@ -42,14 +41,6 @@ public final class KerberosHelper { getKeyTab(user)); } - /** - * Switches user in kerberos. - */ - public static void initUserWithKeytab(String user){ - ExecUtil.executeCommand(new CommandLine("sudo").addArgument("kinit").addArgument(getPrincipal(user)) - .addArgument("-k").addArgument("-t").addArgument(getKeyTab(user))); - } - private static String getKeyTab(String user) { return MerlinConstants.getKeytabForUser(user); } http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java index ae96044..e73bc5d 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/OozieUtil.java @@ -502,29 +502,45 @@ public final class OozieUtil { Assert.assertEquals(actualRun, attempts, "Rerun attempts did not match"); } + /** + * Try to find feed coordinators of given type. + */ + public static int checkIfFeedCoordExist(OozieClient oozieClient, + String feedName, String coordType) throws OozieClientException { + return checkIfFeedCoordExist(oozieClient, feedName, coordType, 5); + } + + /** + * Try to find feed coordinators of given type given number of times. + */ public static int checkIfFeedCoordExist(OozieClient oozieClient, - String feedName, String coordType) throws OozieClientException { + String feedName, String coordType, int numberOfRetries) throws OozieClientException { LOGGER.info("feedName: " + feedName); - int numberOfCoord = 0; - if (getBundles(oozieClient, feedName, EntityType.FEED).size() == 0) { - return 0; - } - List<String> bundleIds = getBundles(oozieClient, feedName, EntityType.FEED); - LOGGER.info("bundleIds: " + bundleIds); - - for (String aBundleId : bundleIds) { - LOGGER.info("aBundleId: " + aBundleId); - waitForCoordinatorJobCreation(oozieClient, aBundleId); - List<CoordinatorJob> coords = - getBundleCoordinators(oozieClient, aBundleId); - LOGGER.info("coords: " + coords); - for (CoordinatorJob coord : coords) { - if (coord.getAppName().contains(coordType)) { - numberOfCoord++; + for (int retryAttempt = 0; retryAttempt < numberOfRetries; retryAttempt++) { + int numberOfCoord = 0; + List<String> bundleIds = getBundles(oozieClient, feedName, EntityType.FEED); + if (bundleIds.size() == 0) { + TimeUtil.sleepSeconds(4); + continue; + } + LOGGER.info("bundleIds: " + bundleIds); + for (String aBundleId : bundleIds) { + LOGGER.info("aBundleId: " + aBundleId); + waitForCoordinatorJobCreation(oozieClient, aBundleId); + List<CoordinatorJob> coords = getBundleCoordinators(oozieClient, aBundleId); + LOGGER.info("coords: " + coords); + for (CoordinatorJob coord : coords) { + if (coord.getAppName().contains(coordType)) { + numberOfCoord++; + } } } + if (numberOfCoord > 0) { + return numberOfCoord; + } + TimeUtil.sleepSeconds(4); } - return numberOfCoord; + return 0; } /** http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java index e5430eb..60eeabd 100644 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java +++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/request/BaseRequest.java @@ -45,6 +45,7 @@ import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.impl.conn.BasicClientConnectionManager; import org.apache.http.message.BasicHeader; +import org.apache.http.util.EntityUtils; import org.apache.log4j.Logger; import javax.net.ssl.SSLContext; @@ -198,6 +199,8 @@ public class BaseRequest { LOGGER.info(String.format("Request Header: Name=%s Value=%s", header.getName(), header.getValue())); } + // Must call this to release the connection + EntityUtils.consume(response.getEntity()); response = client.execute(target, request); } } http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java index 1bffe9a..25cf16f 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/TouchAPIPrismAndServerTest.java @@ -105,6 +105,7 @@ public class TouchAPIPrismAndServerTest extends BaseTestClass { // via server oldbundleId = bundleId; + InstanceUtil.waitTillInstancesAreCreated(clusterOC, bundles[0].getProcessData(), 1); coordId = OozieUtil.getLatestCoordinatorID(clusterOC, bundles[0].getProcessName(), EntityType.PROCESS); response = cluster.getProcessHelper().touchEntity(bundles[0].getProcessData()); bundleId = OozieUtil.getLatestBundleID(clusterOC, bundles[0].getProcessName(), EntityType.PROCESS); http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java index d639c21..49ed94a 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/hcat/HCatRetentionTest.java @@ -262,7 +262,14 @@ public class HCatRetentionTest extends BaseTestClass { //Each HCat partition maps to a directory, not to a file HCatAddPartitionDesc addPtn = HCatAddPartitionDesc.create(dbName, tableName, dataFolder, ptn).build(); - client.addPartition(addPtn); + try { + client.addPartition(addPtn); + } catch (HCatException e) { + //occurrence of specific hive metastore bug + if (!e.getMessage().contains("AlreadyExistsException")) { + throw e; + } + } ptn.clear(); } } http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java index 17725ae..b3ca4f0 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java @@ -69,8 +69,8 @@ public class ListFeedInstancesTest extends BaseTestClass { private String feedDataLocation = sourcePath + MINUTE_DATE_PATTERN; private String targetPath = baseTestHDFSDir + "/target"; private String targetDataLocation = targetPath + MINUTE_DATE_PATTERN; - private final String startTime = "2010-01-02T00:00Z"; - private final String endTime = "2010-01-02T00:57Z"; + private final String startTime = "2015-01-02T00:00Z"; + private final String endTime = "2015-01-02T00:57Z"; private String feedName; @BeforeClass(alwaysRun = true) @@ -210,7 +210,7 @@ public class ListFeedInstancesTest extends BaseTestClass { throws URISyntaxException, OozieClientException, JAXBException, AuthenticationException, IOException, InterruptedException { SoftAssert softAssert = new SoftAssert(); - //orderBy start time, check an order + //orderBy start time, check on order InstancesResult r = prism.getFeedHelper().listInstances(feedName, "orderBy=startTime&sortOrder=desc", null); InstancesResult.Instance[] instances = r.getInstances(); @@ -223,7 +223,7 @@ public class ListFeedInstancesTest extends BaseTestClass { previousDate = (Date) current.clone(); } } - //orderBy status, check an order + //orderBy status, check on order r = prism.getFeedHelper().listInstances(feedName, "start=" + startTime + "&numResults=12&orderBy=status&sortOrder=desc", null); InstanceUtil.validateResponse(r, 12, 1, 1, 8, 2); @@ -235,7 +235,7 @@ public class ListFeedInstancesTest extends BaseTestClass { "Wrong order. Compared " + current + " and " + previousStatus + " statuses."); previousStatus = current; } - //sort by endTime, check an order + //sort by endTime, check on order r = prism.getFeedHelper().listInstances(feedName, "start=" + startTime + "&numResults=12&orderBy=endTime&sortOrder=desc", null); instances = r.getInstances(); @@ -252,7 +252,7 @@ public class ListFeedInstancesTest extends BaseTestClass { } /** - * List instances through api using start/end parameters. Check a number of instances. + * Test the list feed instance api using start/end parameters. Check instances number. */ @Test public void testFeedStartEnd() http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java index 8f45d1c..018b83a 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java @@ -54,6 +54,7 @@ import javax.jms.JMSException; import javax.jms.MapMessage; import java.io.IOException; import java.net.URISyntaxException; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -181,10 +182,13 @@ public class RetentionTest extends BaseTestClass { JmsMessageConsumer messageConsumer = new JmsMessageConsumer("FALCON." + feedName, cluster.getClusterHelper().getActiveMQ()); messageConsumer.start(); - final DateTime currentTime = new DateTime(DateTimeZone.UTC); String bundleId = OozieUtil.getBundles(clusterOC, feedName, EntityType.FEED).get(0); List<String> workflows = OozieUtil.waitForRetentionWorkflowToSucceed(bundleId, clusterOC); + + //get current time minus duration of last status check - to get actual time when eviction has started + final DateTime currentTime = new DateTime(new DateTime(DateTimeZone.UTC).toDate().getTime() - 10000); + LOGGER.info("Current time is " + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(currentTime.toDate())); LOGGER.info("workflows: " + workflows); messageConsumer.interrupt(); Util.printMessageData(messageConsumer); http://git-wip-us.apache.org/repos/asf/falcon/blob/e9e0c7b7/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java index 73273f9..d11411b 100644 --- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java +++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/security/FalconClientTest.java @@ -25,7 +25,6 @@ import org.apache.falcon.regression.core.supportClasses.ExecResult; import org.apache.falcon.regression.core.util.AssertUtil; import org.apache.falcon.regression.core.util.BundleUtil; import org.apache.falcon.regression.core.util.HadoopUtil; -import org.apache.falcon.regression.core.util.KerberosHelper; import org.apache.falcon.regression.core.util.OSUtil; import org.apache.falcon.regression.testHelper.BaseTestClass; import org.apache.hadoop.fs.FileSystem; @@ -81,13 +80,9 @@ public class FalconClientTest extends BaseTestClass { * able to delete * @throws Exception */ - @Test(enabled = true) + @Test(enabled = false) public void badClusterDelete() throws Exception { bundles[0].submitClusters(prism); - //switch user - if (MerlinConstants.IS_SECURE) { - KerberosHelper.initUserWithKeytab(MerlinConstants.DIFFERENT_USER_NAME); - } final String clusterXml = bundles[0].getClusters().get(0); final ExecResult execResult = prism.getClusterHelper().clientDelete(clusterXml, MerlinConstants.DIFFERENT_USER_NAME);
