http://git-wip-us.apache.org/repos/asf/falcon/blob/8e49379d/falcon-regression/README.md ---------------------------------------------------------------------- diff --git a/falcon-regression/README.md b/falcon-regression/README.md deleted file mode 100644 index acb7175..0000000 --- a/falcon-regression/README.md +++ /dev/null @@ -1,291 +0,0 @@ - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -Falcon Regression -================= -This project had 2 modules : - -1. merlin: it has all the system test for falcon -2. merlin-core: it has all the utils used by merlin - -Requirements ------------- -In addition to falcon server and prism, running full falcon regression requires three clusters. -Each of these clusters must have: - -- hadoop -- oozie -- hive -- hcat -For specific tests it may be possible to run it without all clusters and components. - -Prior to running tests Merlin.properties needs to be created and populated with cluster details. - -Configuring Merlin.properties ------------------------------ -Merlin.properties must be created before running falcon regression tests. -The file must be created at the location: - - falcon/falcon-regression/merlin/src/main/resources/Merlin.properties - -Populate it with prism related properties: - - #prism properties - prism.oozie_url = http://node-1.example.com:11000/oozie/ - prism.oozie_location = /usr/lib/oozie/bin - prism.qa_host = node-1.example.com - prism.service_user = falcon - prism.hadoop_url = node-1.example.com:8020 - prism.hadoop_location = /usr/lib/hadoop/bin/hadoop - prism.hostname = http://node-1.example.com:15000 - prism.storeLocation = hdfs://node-1.example.com:8020/apps/falcon - -Specify the clusters that you would be using for testing: - - servers = cluster1,cluster2,cluster3 - -For each cluster specify properties: - - #cluster1 properties - cluster1.oozie_url = http://node-1.example.com:11000/oozie/ - cluster1.oozie_location = /usr/lib/oozie/bin - cluster1.qa_host = node-1.example.com - cluster1.service_user = falcon - cluster1.password = rgautam - cluster1.hadoop_url = node-1.example.com:8020 - cluster1.hadoop_location = /usr/lib/hadoop/bin/hadoop - cluster1.hostname = http://node-1.example.com:15000 - cluster1.cluster_readonly = webhdfs://node-1.example.com:50070 - cluster1.cluster_execute = node-1.example.com:8032 - cluster1.cluster_write = hdfs://node-1.example.com:8020 - cluster1.activemq_url = tcp://node-1.example.com:61616?daemon=true - cluster1.storeLocation = hdfs://node-1.example.com:8020/apps/falcon - cluster1.colo = default - cluster1.namenode.kerberos.principal = nn/node-1.example.com@none - cluster1.hive.metastore.kerberos.principal = hive/node-1.example.com@none - cluster1.hcat_endpoint = thrift://node-1.example.com:9083 - cluster1.service_stop_cmd = /usr/lib/falcon/bin/falcon-stop - cluster1.service_start_cmd = /usr/lib/falcon/bin/falcon-start - -To not clean root tests dir before every test: - - clean_tests_dir=false - -Setting up HDFS Dirs --------------------- -On all cluster as user that started falcon server do: - - hdfs dfs -mkdir -p /tmp/falcon-regression-staging - hdfs dfs -chmod 777 /tmp/falcon-regression-staging - hdfs dfs -mkdir -p /tmp/falcon-regression-working - hdfs dfs -chmod 755 /tmp/falcon-regression-working - -Running Tests -------------- -After creating Merlin.properties file. You can run the following commands to run the tests. - - cd falcon-regression - mvn clean test -Phadoop-2 - -Profiles Supported: hadoop-2 - -To run a specific test: - - mvn clean test -Phadoop-2 -Dtest=EmbeddedPigScriptTest - -If you want to use specific version of any component, they can be specified using -D, for eg: - - mvn clean test -Phadoop-2 -Doozie.version=4.1.0 -Dhadoop.version=2.6.0 - -Security Tests: ---------------- -ACL tests require multiple user account setup: - - other.user.name=root - falcon.super.user.name=falcon - falcon.super2.user.name=falcon2 - -ACL tests also require group name of the current user: - - current_user.group.name=users - -For testing with kerberos set keytabs properties for different users: - - current_user_keytab=/home/qa/hadoopqa/keytabs/qa.headless.keytab - falcon.super.user.keytab=/home/qa/hadoopqa/keytabs/falcon.headless.keytab - falcon.super2.user.keytab=/home/qa/hadoopqa/keytabs/falcon2.headless.keytab - other.user.keytab=/home/qa/hadoopqa/keytabs/root.headless.keytab - -Adding tests to falcon regression: ----------------------------------- -If you wish to contribute to falcon regression, it's as easy as it gets. -All test classes must be added to the directory: - - falcon/falcon-regression/merlin/src/test/java - -This directory contains sub directories such as prism, ui, security, etc -which contain tests specific to these aspects of falcon. Any general test -can be added directly to the parent directory above. If you wish to write -a series of tests for a new feature, feel free to create a new sub directory. -Your test can use the various process/feed/cluster/workflow templates present in: - - falcon/falcon-regression/merlin/src/test/resources - -or you can add your own bundle of XMLs in this directory. Please avoid redundancy of any resource. - -Each test class can contain multiple related tests. Let us look at a sample test class. -*Refer to comments in code for aid* : - -```java - //The License note must be added to each test - - /** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - package org.apache.falcon.regression; - - - import org.apache.falcon.regression.core.bundle.Bundle; - import org.apache.falcon.regression.core.helpers.ColoHelper; - import org.apache.falcon.regression.core.response.ServiceResponse; - import org.apache.falcon.regression.core.util.AssertUtil; - import org.apache.falcon.regression.core.util.BundleUtil; - import org.apache.falcon.regression.testHelper.BaseTestClass; - import org.testng.annotations.AfterMethod; - import org.testng.annotations.BeforeMethod; - import org.testng.annotations.Test; - - @Test(groups = "embedded") - - //Every test class must inherit the BaseTestClass. This class - //helps using properties mentioned in Merlin.properties, in the test. - - public class FeedSubmitTest extends BaseTestClass { - - private ColoHelper cluster = servers.get(0); - private String feed; - - @BeforeMethod(alwaysRun = true) - public void setUp() throws Exception { - - //Several Util classes are available, such as BundleUtil, which for example - //has been used here to read the ELBundle present in falcon/falcon-regression/src/test/resources - - bundles[0] = BundleUtil.readELBundle(); - bundles[0].generateUniqueBundle(); - bundles[0] = new Bundle(bundles[0], cluster); - - //submit the cluster - ServiceResponse response = - prism.getClusterHelper().submitEntity(bundles[0].getClusters().get(0)); - AssertUtil.assertSucceeded(response); - feed = bundles[0].getInputFeedFromBundle(); - } - - @AfterMethod(alwaysRun = true) - public void tearDown() { - removeBundles(); - } - - //Java docs must be added for each test function, explaining what the function does - - /** - * Submit correctly adjusted feed. Response should reflect success. - * - * @throws Exception - */ - @Test(groups = {"singleCluster"}) - public void submitValidFeed() throws Exception { - ServiceResponse response = prism.getFeedHelper().submitEntity(feed); - AssertUtil.assertSucceeded(response); - } - - /** - * Submit and remove feed. Try to submit it again. Response should reflect success. - * - * @throws Exception - */ - @Test(groups = {"singleCluster"}) - public void submitValidFeedPostDeletion() throws Exception { - ServiceResponse response = prism.getFeedHelper().submitEntity(feed); - AssertUtil.assertSucceeded(response); - - response = prism.getFeedHelper().delete(feed); - AssertUtil.assertSucceeded(response); - response = prism.getFeedHelper().submitEntity(feed); - AssertUtil.assertSucceeded(response); - } - } -``` - -* This class, as the name suggests was to test the Feed Submition aspect of Falcon. -It contains multiple test functions, all of which however are various test cases for the same -feature. This organisation in code must be maintained. - -* In order to be able to manipulate feeds, processes and clusters for the various tests, -objects of classes FeedMerlin, ProcessMerlin, ClusterMerlin can be used. There are already existing -functions which use these objects, such as setProcessInput, setFeedValidity, setProcessConcurrency, -setInputFeedPeriodicity etc. in Bundle.java which should serve your purpose well enough. - -* To add more on the utils, you can use functions in HadoopUtil to create HDFS dirs, delete them, -and add data on HDFS, OozieUtil to hit Oozie for checking coordinator/workflow status, TimeUtil to -get lists of dates and directories to aid in data creation, HCatUtil for Hcatalog related utilities, -and many others to make writing tests very easy. - -* Coding conventions are strictly followed. Use the checkstyle xml present in - falcon/checkstyle/src/main/resources/falcon - - in your project to not get checkstyle errors. - -Testing on Windows ------------------- -Some tests switch user to run commands as a different user. Location of binary to switch user is -configurable: - - windows.su.binary=ExecuteAs.exe - -Automatic capture of oozie logs -------------------------------- -For full falcon regression runs. It might be desirable to pull all oozie job -info and logs at the end of the test. This can be done by configuring Merlin.properties: - - log.capture.oozie = true - log.capture.oozie.skip_info = false - log.capture.oozie.skip_log = true - log.capture.location = ../ - -Dumping entities generated by falcon - ------------------------------------- -Add -Dmerlin.dump.staging to the maven command. For example: - - mvn clean test -Phadoop-2 -Dmerlin.dump.staging=true
http://git-wip-us.apache.org/repos/asf/falcon/blob/8e49379d/falcon-regression/merlin-core/pom.xml ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/pom.xml b/falcon-regression/merlin-core/pom.xml deleted file mode 100644 index b1bc609..0000000 --- a/falcon-regression/merlin-core/pom.xml +++ /dev/null @@ -1,249 +0,0 @@ -<!-- - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - --> - -<!-- pom for DataCommons , initinally contains bundle nd util files --> -<project xmlns="http://maven.apache.org/POM/4.0.0"; - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; - xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> - - <parent> - <groupId>org.apache.falcon.regression</groupId> - <artifactId>falcon-regression</artifactId> - <version>0.10-SNAPSHOT</version> - </parent> - <artifactId>falcon-merlin-core</artifactId> - <description>merlin-core - utilities for Apache Falcon regression suite</description> - <name>Apache Falcon Regression Suite Core</name> - <packaging>jar</packaging> - <profiles> - <profile> - <id>hadoop-2</id> - <dependencies> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-auth</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-common</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.hive.hcatalog</groupId> - <artifactId>hive-webhcat-java-client</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.hive.hcatalog</groupId> - <artifactId>hive-hcatalog-core</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-jdbc</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-metastore</artifactId> - </dependency> - </dependencies> - </profile> - </profiles> - <dependencies> - <dependency> - <groupId>org.testng</groupId> - <artifactId>testng</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - </dependency> - - <dependency> - <groupId>commons-lang</groupId> - <artifactId>commons-lang</artifactId> - <version>2.6</version> - </dependency> - - <dependency> - <groupId>org.apache.activemq</groupId> - <artifactId>activemq-all</artifactId> - <version>${activemq.version}</version> - <exclusions> - <exclusion> - <groupId>org.apache.activemq</groupId> - <artifactId>activemq-pool</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.activemq</groupId> - <artifactId>activemq-openwire-legacy</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.activemq</groupId> - <artifactId>activemq-camel</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.activemq</groupId> - <artifactId>activemq-jaas</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.activemq</groupId> - <artifactId>activemq-broker</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.activemq</groupId> - <artifactId>activemq-console</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.activemq</groupId> - <artifactId>activemq-shiro</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.activemq</groupId> - <artifactId>activemq-spring</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.activemq</groupId> - <artifactId>activemq-jms-pool</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.activemq</groupId> - <artifactId>activemq-amqp</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.activemq</groupId> - <artifactId>activemq-mqtt</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.activemq</groupId> - <artifactId>activemq-stomp</artifactId> - </exclusion> - <exclusion> - <groupId>org.apache.activemq</groupId> - <artifactId>activemq-leveldb-store</artifactId> - </exclusion> - <exclusion> - <groupId>org.fusesource.hawtbuf</groupId> - <artifactId>hawtbuf</artifactId> - </exclusion> - </exclusions> - </dependency> - - - <dependency> - <groupId>com.jcraft</groupId> - <artifactId>jsch</artifactId> - </dependency> - - <dependency> - <groupId>com.google.code.gson</groupId> - <artifactId>gson</artifactId> - <scope>compile</scope> - </dependency> - - <dependency> - <groupId>com.googlecode.json-simple</groupId> - <artifactId>json-simple</artifactId> - </dependency> - - <dependency> - <groupId>xml-apis</groupId> - <artifactId>xml-apis</artifactId> - </dependency> - - <dependency> - <groupId>xmlunit</groupId> - <artifactId>xmlunit</artifactId> - </dependency> - - <dependency> - <groupId>javax.xml.bind</groupId> - <artifactId>jaxb-api</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.oozie</groupId> - <artifactId>oozie-client</artifactId> - </dependency> - - <dependency> - <groupId>joda-time</groupId> - <artifactId>joda-time</artifactId> - </dependency> - - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - - <dependency> - <groupId>com.google.code.findbugs</groupId> - <artifactId>annotations</artifactId> - </dependency> - - <dependency> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </dependency> - - <dependency> - <groupId>commons-beanutils</groupId> - <artifactId>commons-beanutils</artifactId> - </dependency> - - <dependency> - <groupId>javax.jms</groupId> - <artifactId>jms</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.falcon</groupId> - <artifactId>falcon-client</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-exec</artifactId> - </dependency> - - <dependency> - <groupId>org.seleniumhq.selenium</groupId> - <artifactId>selenium-firefox-driver</artifactId> - </dependency> - - <dependency> - <groupId>org.seleniumhq.selenium</groupId> - <artifactId>selenium-support</artifactId> - </dependency> - - </dependencies> -</project> http://git-wip-us.apache.org/repos/asf/falcon/blob/8e49379d/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ClusterMerlin.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ClusterMerlin.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ClusterMerlin.java deleted file mode 100644 index 1d25d12..0000000 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ClusterMerlin.java +++ /dev/null @@ -1,325 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.falcon.regression.Entities; - -import org.apache.commons.beanutils.PropertyUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.falcon.entity.v0.EntityType; -import org.apache.falcon.entity.v0.cluster.ACL; -import org.apache.falcon.entity.v0.cluster.Cluster; -import org.apache.falcon.entity.v0.cluster.ClusterLocationType; -import org.apache.falcon.entity.v0.cluster.Interface; -import org.apache.falcon.entity.v0.cluster.Interfaces; -import org.apache.falcon.entity.v0.cluster.Interfacetype; -import org.apache.falcon.entity.v0.cluster.Location; -import org.apache.falcon.entity.v0.cluster.Locations; -import org.apache.falcon.entity.v0.cluster.Properties; -import org.apache.falcon.entity.v0.cluster.Property; -import org.apache.falcon.regression.core.util.Util; -import org.apache.log4j.Logger; -import org.testng.Assert; -import org.testng.asserts.SoftAssert; - -import javax.xml.bind.JAXBException; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** Class for representing a cluster xml. */ -public class ClusterMerlin extends Cluster { - private static final Logger LOGGER = Logger.getLogger(ClusterMerlin.class); - public ClusterMerlin(String clusterData) { - final Cluster cluster = (Cluster) TestEntityUtil.fromString(EntityType.CLUSTER, - clusterData); - try { - PropertyUtils.copyProperties(this, cluster); - } catch (ReflectiveOperationException e) { - Assert.fail("Can't create ClusterMerlin: " + ExceptionUtils.getStackTrace(e)); - } - } - - @Override - public String toString() { - try { - StringWriter sw = new StringWriter(); - EntityType.CLUSTER.getMarshaller().marshal(this, sw); - return sw.toString(); - } catch (JAXBException e) { - throw new RuntimeException(e); - } - } - - /** - * Sets unique names for the cluster. - * @return mapping of old name to new name - * @param prefix prefix of new name - */ - public Map<? extends String, ? extends String> setUniqueName(String prefix) { - final String oldName = getName(); - final String newName = TestEntityUtil.generateUniqueName(prefix, oldName); - setName(newName); - final HashMap<String, String> nameMap = new HashMap<>(1); - nameMap.put(oldName, newName); - return nameMap; - } - - /** - * Set ACL. - */ - public void setACL(String owner, String group, String permission) { - ACL acl = new ACL(); - acl.setOwner(owner); - acl.setGroup(group); - acl.setPermission(permission); - this.setACL(acl); - } - - public void setInterface(Interfacetype interfacetype, String endpoint, String version) { - final Interfaces interfaces = this.getInterfaces(); - final List<Interface> interfaceList = interfaces.getInterfaces(); - for (final Interface anInterface : interfaceList) { - if (anInterface.getType() == interfacetype) { - anInterface.setEndpoint(endpoint); - if (StringUtils.isNotBlank(version)) { - anInterface.setVersion(version); - } - } - } - } - - public void setWorkingLocationPath(String path) { - for (Location location : getLocations().getLocations()) { - if (location.getName() == ClusterLocationType.WORKING) { - location.setPath(path); - break; - } - } - } - - public String getInterfaceEndpoint(final Interfacetype interfaceType) { - String value = null; - for (Interface anInterface : getInterfaces().getInterfaces()) { - if (anInterface.getType() == interfaceType) { - value = anInterface.getEndpoint(); - } - } - LOGGER.info("Cluster: " + getName() + " interfaceType: " + interfaceType - + " value:" + value); - return value; - } - - public String getProperty(final String propName) { - String value = null; - for (Property property : getProperties().getProperties()) { - if (property.getName().trim().equals(propName.trim())) { - value = property.getValue(); - } - } - LOGGER.info("Cluster: " + getName() + " property: " + propName + " value:" + value); - return value; - } - - public String getLocation(final String locationType) { - String value = null; - for (Location location : getLocations().getLocations()) { - if (location.getName().name().trim().equalsIgnoreCase(locationType.trim().toLowerCase())) { - value = location.getPath(); - } - } - LOGGER.info("Cluster: " + getName() + " locationType: " + locationType + " value:" + value); - return value; - } - - /** - * Cleans all properties and returns empty cluster as a draft (as we can't create cluster e.g from empty string). - */ - public ClusterMerlin getEmptyCluster() { - ClusterMerlin clusterMerlin = new ClusterMerlin(this.toString()); - clusterMerlin.setName(""); - clusterMerlin.setDescription(null); - clusterMerlin.setColo(null); - clusterMerlin.setTags(null); - clusterMerlin.setInterfaces(new Interfaces()); - clusterMerlin.setLocations(new Locations()); - clusterMerlin.getACL().setGroup(""); - clusterMerlin.getACL().setOwner(""); - clusterMerlin.setProperties(new Properties()); - return clusterMerlin; - } - - public void addLocation(ClusterLocationType type, String path) { - Location newLocation = new Location(); - newLocation.setName(type); - newLocation.setPath(path); - getLocations().getLocations().add(newLocation); - } - - /** - * Add/replace a property. - * @param name name of the property - * @param value value of the property - * @return this - */ - public ClusterMerlin withProperty(String name, String value) { - final List<Property> properties = getProperties().getProperties(); - //if property with same name exists, just replace the value - for (Property property : properties) { - if (property.getName().equals(name)) { - LOGGER.info(String.format("Overwriting property name = %s oldVal = %s newVal = %s", - property.getName(), property.getValue(), value)); - property.setValue(value); - return this; - } - } - //if property is not added already, add it - final Property property = new Property(); - property.setName(name); - property.setValue(value); - properties.add(property); - return this; - } - - public void addInterface(Interfacetype type, String endpoint, String version) { - Interface iface = new Interface(); - iface.setType(type); - iface.setEndpoint(endpoint); - iface.setVersion(version); - getInterfaces().getInterfaces().add(iface); - } - - public void assertEquals(ClusterMerlin cluster) { - LOGGER.info(String.format("Comparing : source: %n%s%n and cluster: %n%n%s", - Util.prettyPrintXml(toString()), Util.prettyPrintXml(cluster.toString()))); - SoftAssert softAssert = new SoftAssert(); - softAssert.assertEquals(name, cluster.getName(), "Cluster name is different."); - softAssert.assertEquals(colo, cluster.getColo(), "Cluster colo is different."); - softAssert.assertEquals(description, cluster.getDescription(), "Cluster description is different."); - softAssert.assertEquals(tags, cluster.getTags(), "Cluster tags are different."); - softAssert.assertTrue(interfacesEqual(interfaces.getInterfaces(), cluster.getInterfaces().getInterfaces()), - "Cluster interfaces are different"); - softAssert.assertTrue(locationsEqual(locations.getLocations(), cluster.getLocations().getLocations()), - "Cluster locations are different"); - softAssert.assertEquals(acl.getGroup(), cluster.getACL().getGroup(), "Cluster acl group is different."); - softAssert.assertEquals(acl.getOwner(), cluster.getACL().getOwner(), "Cluster acl owner is different."); - softAssert.assertEquals(acl.getPermission(), cluster.getACL().getPermission(), - "Cluster acl permissions is different."); - softAssert.assertTrue(propertiesEqual(properties.getProperties(), cluster.getProperties().getProperties()), - "Cluster properties are different."); - softAssert.assertAll(); - } - - private static boolean checkEquality(String str1, String str2, String message){ - if (!str1.equals(str2)) { - LOGGER.info(String.format("Cluster %s are different: %s and %s.", message, str1, str2)); - return false; - } - return true; - } - - private static boolean interfacesEqual(List<Interface> srcInterfaces, List<Interface> trgInterfaces) { - if (srcInterfaces.size() == trgInterfaces.size()) { - boolean equality = false; - for(Interface iface1: srcInterfaces){ - for(Interface iface2 : trgInterfaces) { - if (iface2.getType().value().equals(iface1.getType().value())) { - equality = checkEquality(iface1.getEndpoint(), iface2.getEndpoint(), - iface1.getType().value() + " interface endpoints"); - equality &= checkEquality(iface1.getVersion(), iface2.getVersion(), - iface1.getType().value() + " interface versions"); - } - } - } - return equality; - } else { - return false; - } - } - - private static boolean propertiesEqual(List<Property> srcProps, List<Property> trgProps) { - if (srcProps.size() == trgProps.size()) { - boolean equality = true; - for(Property prop1: srcProps){ - for(Property prop2 : trgProps) { - if (prop2.getName().equals(prop1.getName())) { - equality &= checkEquality(prop1.getValue(), prop2.getValue(), - prop1.getName() + " property values"); - } - } - } - return equality; - } else { - return false; - } - } - - /** - * Compares two lists of locations. - */ - private static boolean locationsEqual(List<Location> srcLocations, List<Location> objLocations) { - if (srcLocations.size() != objLocations.size()) { - return false; - } - nextType: - for (ClusterLocationType type : ClusterLocationType.values()) { - List<Location> locations1 = new ArrayList<>(); - List<Location> locations2 = new ArrayList<>(); - //get locations of the same type - for (int i = 0; i < srcLocations.size(); i++) { - if (srcLocations.get(i).getName() == type) { - locations1.add(srcLocations.get(i)); - } - if (objLocations.get(i).getName() == type) { - locations2.add(objLocations.get(i)); - } - } - //compare locations of the same type. At least 1 match should be present. - if (locations1.size() != locations2.size()) { - return false; - } - for (Location location1 : locations1) { - for (Location location2 : locations2) { - if (location1.getPath().equals(location2.getPath())) { - continue nextType; - } - } - } - return false; - } - return true; - } - - public Location getLocation(ClusterLocationType type) { - List<Location> locationsOfType = new ArrayList<>(); - for(Location location : locations.getLocations()) { - if (location.getName() == type) { - locationsOfType.add(location); - } - } - Assert.assertEquals(locationsOfType.size(), 1, "Unexpected number of " + type + " locations in: " + this); - return locationsOfType.get(0); - } - @Override - public EntityType getEntityType() { - return EntityType.CLUSTER; - } - -} http://git-wip-us.apache.org/repos/asf/falcon/blob/8e49379d/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java deleted file mode 100644 index ba1d228..0000000 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/FeedMerlin.java +++ /dev/null @@ -1,537 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.falcon.regression.Entities; - -import org.apache.commons.beanutils.PropertyUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.falcon.entity.v0.EntityType; -import org.apache.falcon.entity.v0.Frequency; -import org.apache.falcon.entity.v0.feed.ACL; -import org.apache.falcon.entity.v0.feed.ActionType; -import org.apache.falcon.entity.v0.feed.CatalogTable; -import org.apache.falcon.entity.v0.feed.Cluster; -import org.apache.falcon.entity.v0.feed.ClusterType; -import org.apache.falcon.entity.v0.feed.Feed; -import org.apache.falcon.entity.v0.feed.Location; -import org.apache.falcon.entity.v0.feed.LocationType; -import org.apache.falcon.entity.v0.feed.Locations; -import org.apache.falcon.entity.v0.feed.Property; -import org.apache.falcon.entity.v0.feed.Retention; -import org.apache.falcon.entity.v0.feed.Validity; -import org.apache.falcon.entity.v0.feed.Sla; -import org.apache.falcon.regression.core.util.TimeUtil; -import org.apache.falcon.regression.core.util.Util; -import org.testng.Assert; -import org.apache.log4j.Logger; -import org.testng.asserts.SoftAssert; - - -import javax.xml.bind.JAXBException; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** Class for representing a feed xml. */ -public class FeedMerlin extends Feed { - private static final Logger LOGGER = Logger.getLogger(FeedMerlin.class); - - public FeedMerlin(String feedData) { - this((Feed) TestEntityUtil.fromString(EntityType.FEED, feedData)); - } - - public FeedMerlin(final Feed feed) { - try { - PropertyUtils.copyProperties(this, feed); - this.setACL(feed.getACL()); - } catch (ReflectiveOperationException e) { - Assert.fail("Can't create FeedMerlin: " + ExceptionUtils.getStackTrace(e)); - } - } - - public static List<FeedMerlin> fromString(List<String> feedStrings) { - List<FeedMerlin> feeds = new ArrayList<>(); - for (String feedString : feedStrings) { - feeds.add(fromString(feedString)); - } - return feeds; - } - - public static FeedMerlin fromString(String feedString) { - return new FeedMerlin(feedString); - } - - public List<String> getClusterNames() { - List<String> names = new ArrayList<>(); - for (Cluster cluster : getClusters().getClusters()) { - names.add(cluster.getName()); - } - return names; - } - - /** - * Add/replace a property. - * @param name name of the property - * @param value value of the property - * @return this - */ - public FeedMerlin withProperty(String name, String value) { - final List<Property> properties = getProperties().getProperties(); - //if property with same name exists, just replace the value - for (Property property : properties) { - if (property.getName().equals(name)) { - LOGGER.info(String.format("Overwriting property name = %s oldVal = %s newVal = %s", - property.getName(), property.getValue(), value)); - property.setValue(value); - return this; - } - } - //if property is not added already, add it - final Property property = new Property(); - property.setName(name); - property.setValue(value); - properties.add(property); - return this; - } - - /** - * Return feed path of the specified type. - * @return feed data path - * @param locationType the type of the location - */ - public String getFeedPath(LocationType locationType) { - for (Location location : this.getLocations().getLocations()) { - if (location.getType() == locationType) { - return location.getPath(); - } - } - return null; - } - - /** - * Sets cut-off period. - * @param frequency cut-off period - */ - public FeedMerlin insertLateFeedValue(Frequency frequency) { - this.getLateArrival().setCutOff(frequency); - return this; - } - - /** - * Sets data location for a feed. - * @param pathValue new path - */ - public FeedMerlin setFeedPathValue(String pathValue) { - for (Location location : this.getLocations().getLocations()) { - if (location.getType() == LocationType.DATA) { - location.setPath(pathValue); - } - } - return this; - } - - /** - * Sets name for a cluster by given order number. - * @param clusterName new cluster name - * @param clusterIndex index of cluster which should be updated - */ - public FeedMerlin setClusterNameInFeed(String clusterName, int clusterIndex) { - this.getClusters().getClusters().get(clusterIndex).setName(clusterName); - return this; - } - - /** clear clusters of this feed. */ - public FeedMerlin clearFeedClusters() { - getClusters().getClusters().clear(); - return this; - } - - /** add a feed cluster to this feed. */ - public FeedMerlin addFeedCluster(Cluster cluster) { - getClusters().getClusters().add(cluster); - return this; - } - - /** Fluent builder wrapper for cluster fragment of feed entity . */ - public static class FeedClusterBuilder { - private Cluster cluster = new Cluster(); - - public FeedClusterBuilder(String clusterName) { - cluster.setName(clusterName); - } - - public Cluster build() { - Cluster retVal = cluster; - cluster = null; - return retVal; - } - - public FeedClusterBuilder withRetention(String limit, ActionType action) { - Retention r = new Retention(); - r.setLimit(new Frequency(limit)); - r.setAction(action); - cluster.setRetention(r); - return this; - } - - public FeedClusterBuilder withValidity(String startTime, String endTime) { - Validity v = new Validity(); - v.setStart(TimeUtil.oozieDateToDate(startTime).toDate()); - v.setEnd(TimeUtil.oozieDateToDate(endTime).toDate()); - cluster.setValidity(v); - return this; - } - - public FeedClusterBuilder withClusterType(ClusterType type) { - cluster.setType(type); - return this; - } - - public FeedClusterBuilder withPartition(String partition) { - cluster.setPartition(partition); - return this; - } - - public FeedClusterBuilder withTableUri(String tableUri) { - CatalogTable catalogTable = new CatalogTable(); - catalogTable.setUri(tableUri); - cluster.setTable(catalogTable); - return this; - } - - public FeedClusterBuilder withDataLocation(String dataLocation) { - Location oneLocation = new Location(); - oneLocation.setPath(dataLocation); - oneLocation.setType(LocationType.DATA); - - Locations feedLocations = new Locations(); - feedLocations.getLocations().add(oneLocation); - cluster.setLocations(feedLocations); - return this; - } - - public FeedClusterBuilder withDelay(Frequency frequency) { - cluster.setDelay(frequency); - return this; - } - - - } - - /** - * Method sets a number of clusters to feed definition. - * - * @param newClusters list of definitions of clusters which are to be set to feed - * @param location location of data on every cluster - * @param startTime start of feed validity on every cluster - * @param endTime end of feed validity on every cluster - */ - public void setFeedClusters(List<String> newClusters, String location, String startTime, - String endTime) { - clearFeedClusters(); - setFrequency(new Frequency("" + 5, Frequency.TimeUnit.minutes)); - - for (String newCluster : newClusters) { - Cluster feedCluster = new FeedClusterBuilder(new ClusterMerlin(newCluster).getName()) - .withDataLocation(location + "/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}") - .withValidity(TimeUtil.addMinsToTime(startTime, -180), - TimeUtil.addMinsToTime(endTime, 180)) - .withRetention("hours(20)", ActionType.DELETE) - .build(); - addFeedCluster(feedCluster); - } - } - - public void setRetentionValue(String retentionValue) { - for (org.apache.falcon.entity.v0.feed.Cluster cluster : getClusters().getClusters()) { - cluster.getRetention().setLimit(new Frequency(retentionValue)); - } - } - - public void setTableValue(String dBName, String tableName, String pathValue) { - getTable().setUri("catalog:" + dBName + ":" + tableName + "#" + pathValue); - } - - @Override - public String toString() { - try { - StringWriter sw = new StringWriter(); - EntityType.FEED.getMarshaller().marshal(this, sw); - return sw.toString(); - } catch (JAXBException e) { - throw new RuntimeException(e); - } - } - - public void setLocation(LocationType locationType, String feedInputPath) { - for (Location location : getLocations().getLocations()) { - if (location.getType() == locationType) { - location.setPath(feedInputPath); - } - } - } - - /** - * Sets unique names for the feed. - * @return mapping of old name to new name - * @param prefix prefix of new name - */ - public Map<? extends String, ? extends String> setUniqueName(String prefix) { - final String oldName = getName(); - final String newName = TestEntityUtil.generateUniqueName(prefix, oldName); - setName(newName); - final HashMap<String, String> nameMap = new HashMap<>(1); - nameMap.put(oldName, newName); - return nameMap; - } - - public void renameClusters(Map<String, String> clusterNameMap) { - for (Cluster cluster : getClusters().getClusters()) { - final String oldName = cluster.getName(); - final String newName = clusterNameMap.get(oldName); - if (!StringUtils.isEmpty(newName)) { - cluster.setName(newName); - } - } - } - - /** - * Set ACL. - */ - public void setACL(String owner, String group, String permission) { - ACL acl = new ACL(); - acl.setOwner(owner); - acl.setGroup(group); - acl.setPermission(permission); - this.setACL(acl); - } - - /** - * Sel SLA. - * @param slaLow : low value of SLA - * @param slaHigh : high value of SLA - */ - - public void setSla(Frequency slaLow, Frequency slaHigh) { - Sla sla = new Sla(); - sla.setSlaLow(slaLow); - sla.setSlaHigh(slaHigh); - this.setSla(sla); - } - - /** - * Sets new feed data path (for first location). - * - * @param path new feed data path - */ - public void setFilePath(String path) { - getLocations().getLocations().get(0).setPath(path); - } - - - /** - * Retrieves prefix (main sub-folders) of first feed data path. - */ - public String getFeedPrefix() { - String path = getLocations().getLocations().get(0).getPath(); - return path.substring(0, path.indexOf('$')); - } - - public void setValidity(String feedStart, String feedEnd) { - this.getClusters().getClusters().get(0).getValidity() - .setStart(TimeUtil.oozieDateToDate(feedStart).toDate()); - this.getClusters().getClusters().get(0).getValidity() - .setEnd(TimeUtil.oozieDateToDate(feedEnd).toDate()); - - } - - public void setDataLocationPath(String path) { - final List<Location> locations = this.getLocations().getLocations(); - for (Location location : locations) { - if (location.getType() == LocationType.DATA) { - location.setPath(path); - } - } - } - - public void setPeriodicity(int frequency, Frequency.TimeUnit periodicity) { - Frequency frq = new Frequency(String.valueOf(frequency), periodicity); - this.setFrequency(frq); - } - - public void setTableUri(String tableUri) { - final CatalogTable catalogTable = new CatalogTable(); - catalogTable.setUri(tableUri); - this.setTable(catalogTable); - } - - @Override - public EntityType getEntityType() { - return EntityType.FEED; - } - - public void assertGeneralProperties(FeedMerlin newFeed){ - - LOGGER.info(String.format("Comparing General Properties: source: %n%s%n and feed: %n%n%s", - Util.prettyPrintXml(toString()), Util.prettyPrintXml(newFeed.toString()))); - - SoftAssert softAssert = new SoftAssert(); - - // Assert all the the General Properties - softAssert.assertEquals(newFeed.getName(), getName(), - "Feed Name is different"); - softAssert.assertEquals(newFeed.getDescription(), getDescription(), - "Feed Description is different"); - softAssert.assertEquals(newFeed.getTags(), getTags(), - "Feed Tags is different"); - softAssert.assertEquals(newFeed.getGroups(), getGroups(), - "Feed Groups is different"); - softAssert.assertEquals(newFeed.getACL().getOwner(), getACL().getOwner(), - "Feed ACL Owner is different"); - softAssert.assertEquals(newFeed.getACL().getGroup(), getACL().getGroup(), - "Feed ACL Group is different"); - softAssert.assertEquals(newFeed.getACL().getPermission(), getACL().getPermission(), - "Feed ACL Permission is different"); - softAssert.assertEquals(newFeed.getSchema().getLocation(), getSchema().getLocation(), - "Feed Schema Location is different"); - softAssert.assertEquals(newFeed.getSchema().getProvider(), getSchema().getProvider(), - "Feed Schema Provider is different"); - softAssert.assertAll(); - - } - - public void assertPropertiesInfo(FeedMerlin newFeed){ - - LOGGER.info(String.format("Comparing Properties Info: source: %n%s%n and feed: %n%n%s", - Util.prettyPrintXml(toString()), Util.prettyPrintXml(newFeed.toString()))); - - SoftAssert softAssert = new SoftAssert(); - - // Assert all the Properties Info - softAssert.assertEquals(newFeed.getFrequency().getFrequency(), - getFrequency().getFrequency(), - "Feed Frequency is different"); - softAssert.assertEquals(newFeed.getFrequency().getTimeUnit().toString(), - getFrequency().getTimeUnit().toString(), - "Feed Frequency Unit is different"); - softAssert.assertEquals(newFeed.getLateArrival().getCutOff().getFrequencyAsInt(), - getLateArrival().getCutOff().getFrequencyAsInt(), - "Feed CutOff is different"); - softAssert.assertEquals(newFeed.getLateArrival().getCutOff().getTimeUnit(), - getLateArrival().getCutOff().getTimeUnit(), - "Feed CutOff Unit is different"); - softAssert.assertEquals(newFeed.getAvailabilityFlag(), - getAvailabilityFlag(), - "Feed Availability Flag is different"); - softAssert.assertEquals(newFeed.getProperties().getProperties().get(0).getName(), - getProperties().getProperties().get(0).getName(), - "Feed Property1 Name is different"); - softAssert.assertEquals(newFeed.getProperties().getProperties().get(0).getValue(), - getProperties().getProperties().get(0).getValue(), - "Feed Property1 Value is different"); - softAssert.assertEquals(newFeed.getProperties().getProperties().get(1).getName(), - getProperties().getProperties().get(1).getName(), - "Feed Property2 Name is different"); - softAssert.assertEquals(newFeed.getProperties().getProperties().get(1).getValue(), - getProperties().getProperties().get(1).getValue(), - "Feed Property2 Value is different"); - - softAssert.assertAll(); - } - - public void assertLocationInfo(FeedMerlin newFeed){ - - LOGGER.info(String.format("Comparing Location Info: source: %n%s%n and feed: %n%n%s", - Util.prettyPrintXml(toString()), Util.prettyPrintXml(newFeed.toString()))); - - SoftAssert softAssert = new SoftAssert(); - - // Assert all the Location Properties - softAssert.assertEquals(newFeed.getLocations().getLocations().get(0).getPath(), - getLocations().getLocations().get(0).getPath(), - "Feed Location Data Path is different"); - softAssert.assertEquals(newFeed.getLocations().getLocations().get(1).getPath(), - getLocations().getLocations().get(1).getPath(), - "Feed Location Stats Path is different"); - softAssert.assertEquals(newFeed.getLocations().getLocations().get(2).getPath(), - getLocations().getLocations().get(2).getPath(), - "Feed Location Meta Path is different"); - - softAssert.assertAll(); - - } - - public void assertClusterInfo(FeedMerlin newFeed){ - - LOGGER.info(String.format("Comparing Feed Cluster Info: source: %n%s%n and feed: %n%n%s", - Util.prettyPrintXml(toString()), Util.prettyPrintXml(newFeed.toString()))); - - SoftAssert softAssert = new SoftAssert(); - - // Assert all the Cluster Properties - softAssert.assertEquals(newFeed.getClusters().getClusters().get(0) - .getName(), - getClusters().getClusters().get(0).getName(), - "Feed Cluster Name is different"); - softAssert.assertEquals(newFeed.getClusters().getClusters().get(0) - .getLocations().getLocations().get(0).getPath(), - getLocations().getLocations().get(0).getPath(), - "Feed Cluster Data Path is different"); - softAssert.assertEquals(newFeed.getClusters().getClusters().get(0) - .getLocations().getLocations().get(1).getPath(), - getLocations().getLocations().get(1).getPath(), - "Feed Cluster Stats Path is different"); - softAssert.assertEquals(newFeed.getClusters().getClusters().get(0) - .getLocations().getLocations().get(2).getPath(), - getLocations().getLocations().get(2).getPath(), - "Feed Cluster Meta Path is different"); - softAssert.assertEquals(newFeed.getClusters().getClusters().get(0) - .getValidity().getStart(), - getClusters().getClusters().get(0).getValidity().getStart(), - "Feed Cluster Start Date is different"); - softAssert.assertEquals(newFeed.getClusters().getClusters().get(0) - .getValidity().getEnd(), - getClusters().getClusters().get(0).getValidity().getEnd(), - "Feed Cluster End Date is different"); - // Asserting on hardcoded value of 99, due to UI bug which only support till two digits. - softAssert.assertEquals(newFeed.getClusters().getClusters().get(0) - .getRetention().getLimit().getFrequency(), "99", - "Feed Retention is different"); - softAssert.assertEquals(newFeed.getClusters().getClusters().get(0) - .getRetention().getLimit().getTimeUnit().name(), - getClusters().getClusters().get(0).getRetention().getLimit().getTimeUnit().name(), - "Feed Retention Unit is different"); - - softAssert.assertAll(); - - } - - public void assertEquals(FeedMerlin newFeed) { - - assertGeneralProperties(newFeed); - assertPropertiesInfo(newFeed); - assertLocationInfo(newFeed); - assertClusterInfo(newFeed); - } - - - -} http://git-wip-us.apache.org/repos/asf/falcon/blob/8e49379d/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java deleted file mode 100644 index fb76da1..0000000 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/ProcessMerlin.java +++ /dev/null @@ -1,691 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.falcon.regression.Entities; - -import org.apache.commons.beanutils.PropertyUtils; -import org.apache.commons.lang.ArrayUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.falcon.entity.v0.EntityType; -import org.apache.falcon.entity.v0.Frequency; -import org.apache.falcon.entity.v0.process.ACL; -import org.apache.falcon.entity.v0.process.Cluster; -import org.apache.falcon.entity.v0.process.EngineType; -import org.apache.falcon.entity.v0.process.Input; -import org.apache.falcon.entity.v0.process.Inputs; -import org.apache.falcon.entity.v0.process.Output; -import org.apache.falcon.entity.v0.process.Outputs; -import org.apache.falcon.entity.v0.process.Process; -import org.apache.falcon.entity.v0.process.Properties; -import org.apache.falcon.entity.v0.process.Property; -import org.apache.falcon.entity.v0.process.Sla; -import org.apache.falcon.entity.v0.process.Validity; -import org.apache.falcon.entity.v0.process.Workflow; -import org.apache.falcon.regression.core.util.TimeUtil; -import org.apache.falcon.regression.core.util.Util; -import org.apache.log4j.Logger; -import org.testng.Assert; -import org.testng.asserts.SoftAssert; - -import javax.xml.bind.JAXBException; -import java.io.StringWriter; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -/** Class for representing a process xml. */ -public class ProcessMerlin extends Process { - private static final Logger LOGGER = Logger.getLogger(ProcessMerlin.class); - public ProcessMerlin(String processData) { - this((Process) TestEntityUtil.fromString(EntityType.PROCESS, processData)); - } - - public ProcessMerlin(final Process process) { - try { - PropertyUtils.copyProperties(this, process); - } catch (ReflectiveOperationException e) { - Assert.fail("Can't create ProcessMerlin: " + ExceptionUtils.getStackTrace(e)); - } - } - - public ProcessMerlin clearProcessCluster() { - getClusters().getClusters().clear(); - return this; - } - - public ProcessMerlin addProcessCluster(Cluster cluster) { - getClusters().getClusters().add(cluster); - return this; - } - - public List<String> getClusterNames() { - List<String> names = new ArrayList<>(); - for (Cluster cluster : getClusters().getClusters()) { - names.add(cluster.getName()); - } - return names; - } - - public Cluster getClusterByName(String name) { - for (Cluster cluster : getClusters().getClusters()) { - if (name.equals(cluster.getName())) { - return cluster; - } - } - return null; - } - - /** - * Compares two process cluster lists, if they are equal or not. - */ - public static void assertClustersEqual(List<Cluster> clusters1, List<Cluster> clusters2) { - if (clusters1.size() != clusters2.size()) { - Assert.fail("Cluster sizes are different."); - } - Comparator<Cluster> clusterComparator = new Comparator<Cluster>() { - @Override - public int compare(Cluster cluster1, Cluster cluster2) { - return cluster1.getName().compareTo(cluster2.getName()); - } - }; - Collections.sort(clusters1, clusterComparator); - Collections.sort(clusters2, clusterComparator); - SoftAssert softAssert = new SoftAssert(); - for(int i = 0; i < clusters1.size(); i++) { - Cluster cluster1 = clusters1.get(i); - Cluster cluster2 = clusters2.get(i); - softAssert.assertEquals(cluster1.getName(), cluster2.getName(), "Cluster names are different."); - softAssert.assertEquals(cluster1.getValidity().getStart(), cluster2.getValidity().getStart(), - String.format("Validity start is not the same for cluster %s", cluster1.getName())); - softAssert.assertEquals(cluster1.getValidity().getEnd(), cluster2.getValidity().getEnd(), - String.format("Cluster validity end is not the same for cluster %s", cluster1.getName())); - } - softAssert.assertAll(); - } - - public Input getInputByName(String name) { - for (Input input : getInputs().getInputs()) { - if (input.getName().equals(name)) { - return input; - } - } - return null; - } - - public Output getOutputByName(String name) { - for (Output output : getOutputs().getOutputs()) { - if (output.getName().equals(name)) { - return output; - } - } - return null; - } - - /** Fluent builder wrapper for cluster fragment of process entity . */ - public static class ProcessClusterBuilder { - private Cluster cluster = new Cluster(); - - public ProcessClusterBuilder(String clusterName) { - cluster.setName(clusterName); - } - - public Cluster build() { - Cluster retVal = cluster; - cluster = null; - return retVal; - } - - public ProcessClusterBuilder withValidity(String startTime, String endTime) { - Validity v = new Validity(); - v.setStart(TimeUtil.oozieDateToDate(startTime).toDate()); - v.setEnd(TimeUtil.oozieDateToDate(endTime).toDate()); - cluster.setValidity(v); - return this; - } - - } - - /** - * Method sets a number of clusters to process definition. - * - * @param newClusters list of definitions of clusters which are to be set to process - * (clusters on which process should run) - * @param startTime start of process validity on every cluster - * @param endTime end of process validity on every cluster - */ - public void setProcessClusters(List<String> newClusters, String startTime, String endTime) { - clearProcessCluster(); - for (String newCluster : newClusters) { - final Cluster processCluster = new ProcessClusterBuilder( - new ClusterMerlin(newCluster).getName()) - .withValidity(startTime, endTime) - .build(); - addProcessCluster(processCluster); - } - } - - public final ProcessMerlin clearProperties() { - final Properties properties = new Properties(); - setProperties(properties); - return this; - } - - /** - * Add/replace a property. - * @param name name of the property - * @param value value of the property - * @return this - */ - public final ProcessMerlin withProperty(String name, String value) { - final List<Property> properties = getProperties().getProperties(); - //if property with same name exists, just replace the value - for (Property property : properties) { - if (property.getName().equals(name)) { - LOGGER.info(String.format("Overwriting property name = %s oldVal = %s newVal = %s", - property.getName(), property.getValue(), value)); - property.setValue(value); - return this; - } - } - //if property is not added already, add it - final Property property = new Property(); - property.setName(name); - property.setValue(value); - properties.add(property); - return this; - } - - public String getProperty(String name) { - for (Property property : properties.getProperties()) { - if (property.getName().equals(name)) { - return property.getValue(); - } - } - return null; - } - - @Override - public String toString() { - try { - StringWriter sw = new StringWriter(); - EntityType.PROCESS.getMarshaller().marshal(this, sw); - return sw.toString(); - } catch (JAXBException e) { - throw new RuntimeException(e); - } - } - - public void renameClusters(Map<String, String> clusterNameMap) { - for (Cluster cluster : getClusters().getClusters()) { - final String oldName = cluster.getName(); - final String newName = clusterNameMap.get(oldName); - if (!StringUtils.isEmpty(newName)) { - cluster.setName(newName); - } - } - } - - public void renameFeeds(Map<String, String> feedNameMap) { - for(Input input : getInputs().getInputs()) { - final String oldName = input.getFeed(); - final String newName = feedNameMap.get(oldName); - if (!StringUtils.isEmpty(newName)) { - input.setFeed(newName); - } - } - for(Output output : getOutputs().getOutputs()) { - final String oldName = output.getFeed(); - final String newName = feedNameMap.get(oldName); - if (!StringUtils.isEmpty(newName)) { - output.setFeed(newName); - } - } - } - - /** - * Sets unique names for the process. - * @return mapping of old name to new name - * @param prefix prefix of new name - */ - public Map<? extends String, ? extends String> setUniqueName(String prefix) { - final String oldName = getName(); - final String newName = TestEntityUtil.generateUniqueName(prefix, oldName); - setName(newName); - final HashMap<String, String> nameMap = new HashMap<>(1); - nameMap.put(oldName, newName); - return nameMap; - } - - /** - * Method sets optional/compulsory inputs and outputs of process according to list of feed - * definitions and matching numeric parameters. Optional inputs are set first and then - * compulsory ones. - * - * @param newDataSets list of feed definitions - * @param numberOfInputs number of desired inputs - * @param numberOfOptionalInput how many inputs should be optional - * @param numberOfOutputs number of outputs - */ - public void setProcessFeeds(List<String> newDataSets, - int numberOfInputs, int numberOfOptionalInput, - int numberOfOutputs) { - int numberOfOptionalSet = 0; - boolean isFirst = true; - - Inputs is = new Inputs(); - for (int i = 0; i < numberOfInputs; i++) { - Input in = new Input(); - in.setEnd("now(0,0)"); - in.setStart("now(0,-20)"); - if (numberOfOptionalSet < numberOfOptionalInput) { - in.setOptional(true); - in.setName("inputData" + i); - numberOfOptionalSet++; - } else { - in.setOptional(false); - if (isFirst) { - in.setName("inputData"); - isFirst = false; - } else { - in.setName("inputData" + i); - } - } - in.setFeed(new FeedMerlin(newDataSets.get(i)).getName()); - is.getInputs().add(in); - } - - setInputs(is); - if (numberOfInputs == 0) { - setInputs(null); - } - - Outputs os = new Outputs(); - for (int i = 0; i < numberOfOutputs; i++) { - Output op = new Output(); - op.setFeed(new FeedMerlin(newDataSets.get(numberOfInputs - i)).getName()); - op.setName("outputData"); - op.setInstance("now(0,0)"); - os.getOutputs().add(op); - } - setOutputs(os); - setLateProcess(null); - } - - /** - * Sets process pipelines tag. - * @param pipelines set of pipelines to be set to process - */ - public void setPipelineTag(String... pipelines){ - if (ArrayUtils.isNotEmpty(pipelines)){ - this.pipelines = StringUtils.join(pipelines, ","); - } else { - this.pipelines = null; - } - } - - /** - * Set ACL. - */ - public void setACL(String owner, String group, String permission) { - ACL acl = new ACL(); - acl.setOwner(owner); - acl.setGroup(group); - acl.setPermission(permission); - this.setACL(acl); - } - - /** - * Set SLA. - * @param slaStart : start value of SLA - * @param slaEnd : end value of SLA - */ - - public void setSla(Frequency slaStart, Frequency slaEnd) { - Sla sla = new Sla(); - sla.setShouldStartIn(slaStart); - sla.setShouldEndIn(slaEnd); - this.setSla(sla); - } - - /** - * Sets new process validity on all the process clusters. - * - * @param startTime start of process validity - * @param endTime end of process validity - */ - public void setValidity(String startTime, String endTime) { - - for (Cluster cluster : this.getClusters().getClusters()) { - cluster.getValidity().setStart(TimeUtil.oozieDateToDate(startTime).toDate()); - cluster.getValidity().setEnd(TimeUtil.oozieDateToDate(endTime).toDate()); - } - } - - /** - * Adds one output into process. - */ - public void addOutputFeed(String outputName, String feedName) { - Output out1 = getOutputs().getOutputs().get(0); - Output out2 = new Output(); - out2.setFeed(feedName); - out2.setName(outputName); - out2.setInstance(out1.getInstance()); - getOutputs().getOutputs().add(out2); - } - - /** - * Adds one input into process. - */ - public void addInputFeed(String inputName, String feedName) { - Input in1 = getInputs().getInputs().get(0); - Input in2 = new Input(); - in2.setEnd(in1.getEnd()); - in2.setFeed(feedName); - in2.setName(inputName); - in2.setPartition(in1.getPartition()); - in2.setStart(in1.getStart()); - in2.setOptional(in1.isOptional()); - getInputs().getInputs().add(in2); - } - - public void setInputFeedWithEl(String inputFeedName, String startEl, String endEl) { - Inputs inputs = new Inputs(); - Input input = new Input(); - input.setFeed(inputFeedName); - input.setStart(startEl); - input.setEnd(endEl); - input.setName("inputData"); - inputs.getInputs().add(input); - this.setInputs(inputs); - } - - public void setDatasetInstances(String startInstance, String endInstance) { - this.getInputs().getInputs().get(0).setStart(startInstance); - this.getInputs().getInputs().get(0).setEnd(endInstance); - } - - public void setProcessInputStartEnd(String start, String end) { - for (Input input : this.getInputs().getInputs()) { - input.setStart(start); - input.setEnd(end); - } - } - - /** - * Sets name(s) of the process output(s). - * - * @param names new names of the outputs - */ - public void setOutputNames(String... names) { - Outputs outputs = this.getOutputs(); - Assert.assertEquals(outputs.getOutputs().size(), names.length, - "Number of output names is not equal to number of outputs in process"); - for (int i = 0; i < names.length; i++) { - outputs.getOutputs().get(i).setName(names[i]); - } - this.setOutputs(outputs); - } - - /** - * Sets partition for each input, according to number of supplied partitions. - * - * @param partition partitions to be set - */ - public void setInputPartition(String... partition) { - for (int i = 0; i < partition.length; i++) { - this.getInputs().getInputs().get(i).setPartition(partition[i]); - } - } - - /** - * Changes names of process inputs. - * - * @param names desired names of inputs - */ - public void setInputNames(String... names) { - for (int i = 0; i < names.length; i++) { - this.getInputs().getInputs().get(i).setName(names[i]); - } - } - - public void setPeriodicity(int frequency, Frequency.TimeUnit periodicity) { - Frequency frq = new Frequency(String.valueOf(frequency), periodicity); - this.setFrequency(frq); - } - - public void setTimeOut(int magnitude, Frequency.TimeUnit unit) { - Frequency frq = new Frequency(String.valueOf(magnitude), unit); - this.setTimeout(frq); - } - - public void setWorkflow(String wfPath, String libPath, EngineType engineType) { - Workflow w = this.getWorkflow(); - if (engineType != null) { - w.setEngine(engineType); - } - if (libPath != null) { - w.setLib(libPath); - } - w.setPath(wfPath); - this.setWorkflow(w); - } - - public String getFirstInputName() { - return getInputs().getInputs().get(0).getName(); - } - - @Override - public EntityType getEntityType() { - return EntityType.PROCESS; - } - - public void assertGeneralProperties(ProcessMerlin newProcess){ - SoftAssert softAssert = new SoftAssert(); - // Assert all the the General Properties - softAssert.assertEquals(newProcess.getName(), getName(), - "Process Name is different"); - softAssert.assertEquals(newProcess.getTags(), getTags(), - "Process Tags Value is different"); - softAssert.assertEquals(newProcess.getWorkflow().getName(), getWorkflow().getName(), - "Process Workflow Name is different"); - if (getWorkflow().getEngine() == EngineType.OOZIE || getWorkflow().getEngine() == null) { - softAssert.assertTrue(newProcess.getWorkflow().getEngine() == EngineType.OOZIE - || newProcess.getWorkflow().getEngine() == null, "Process Workflow Engine is different"); - } else { - softAssert.assertEquals(newProcess.getWorkflow().getEngine().toString(), - getWorkflow().getEngine().toString(), - "Process Workflow Engine is different"); - } - softAssert.assertEquals(newProcess.getWorkflow().getPath(), getWorkflow().getPath(), - "Process Workflow Path is different"); - softAssert.assertEquals(newProcess.getACL().getOwner(), getACL().getOwner(), - "Process ACL Owner is different"); - softAssert.assertEquals(newProcess.getACL().getGroup(), getACL().getGroup(), - "Process ACL Group is different"); - softAssert.assertEquals(newProcess.getACL().getPermission(), getACL().getPermission(), - "Process ACL Permission is different"); - softAssert.assertAll(); - } - - public void assertPropertiesInfo(ProcessMerlin newProcess){ - SoftAssert softAssert = new SoftAssert(); - // Assert all the Properties Info - softAssert.assertEquals(newProcess.getTimezone().getID(), getTimezone().getID(), - "Process TimeZone is different"); - softAssert.assertEquals(newProcess.getFrequency().getFrequency(), getFrequency().getFrequency(), - "Process Frequency is different"); - softAssert.assertEquals(newProcess.getFrequency().getTimeUnit().toString(), - getFrequency().getTimeUnit().toString(), - "Process Frequency Unit is different"); - softAssert.assertEquals(newProcess.getParallel(), getParallel(), - "Process Parallel is different"); - softAssert.assertEquals(newProcess.getOrder(), getOrder(), - "Process Order is different"); - softAssert.assertEquals(newProcess.getRetry().getPolicy().value(), - getRetry().getPolicy().value(), - "Process Retry Policy is different"); - softAssert.assertEquals(newProcess.getRetry().getAttempts(), - getRetry().getAttempts(), - "Process Retry Attempts is different"); - softAssert.assertEquals(newProcess.getRetry().getDelay().getFrequency(), - getRetry().getDelay().getFrequency(), - "Process Delay Frequency is different"); - softAssert.assertEquals(newProcess.getRetry().getDelay().getTimeUnit().name(), - getRetry().getDelay().getTimeUnit().name(), - "Process Delay Unit is different"); - softAssert.assertAll(); - } - - /** - * Asserts equality of process inputs. - */ - public void assertInputValues(ProcessMerlin newProcess){ - Assert.assertEquals(newProcess.getInputs().getInputs().size(), getInputs().getInputs().size(), - "Processes have different number of inputs."); - SoftAssert softAssert = new SoftAssert(); - // Assert all the Input values - for (int i = 0; i < newProcess.getInputs().getInputs().size(); i++) { - softAssert.assertEquals(newProcess.getInputs().getInputs().get(i).getName(), - getInputs().getInputs().get(i).getName(), - "Process Input Name is different"); - softAssert.assertEquals(newProcess.getInputs().getInputs().get(i).getFeed(), - getInputs().getInputs().get(i).getFeed(), - "Process Input Feed is different"); - softAssert.assertEquals(newProcess.getInputs().getInputs().get(i).getStart(), - getInputs().getInputs().get(i).getStart(), - "Process Input Start is different"); - softAssert.assertEquals(newProcess.getInputs().getInputs().get(i).getEnd(), - getInputs().getInputs().get(i).getEnd(), - "Process Input End is different"); - softAssert.assertEquals(newProcess.getInputs().getInputs().get(i).isOptional(), - getInputs().getInputs().get(i).isOptional(), - "Process Input optional param is different"); - } - softAssert.assertAll(); - } - - /** - * Asserts equality of process outputs. - */ - public void assertOutputValues(ProcessMerlin newProcess){ - SoftAssert softAssert = new SoftAssert(); - // Assert all the Output values - softAssert.assertEquals(newProcess.getOutputs().getOutputs().get(0).getName(), - getOutputs().getOutputs().get(0).getName(), - "Process Output Name is different"); - softAssert.assertEquals(newProcess.getOutputs().getOutputs().get(0).getFeed(), - getOutputs().getOutputs().get(0).getFeed(), - "Process Output Feed is different"); - softAssert.assertEquals(newProcess.getOutputs().getOutputs().get(0).getInstance(), - getOutputs().getOutputs().get(0).getInstance(), - "Process Output Instance is different"); - softAssert.assertAll(); - } - - /** - * Asserts equality of two processes. - */ - public void assertEquals(ProcessMerlin process) { - LOGGER.info(String.format("Comparing General Properties: source: %n%s%n and process: %n%n%s", - Util.prettyPrintXml(toString()), Util.prettyPrintXml(process.toString()))); - assertGeneralProperties(process); - assertInputValues(process); - assertOutputValues(process); - assertPropertiesInfo(process); - assertClustersEqual(getClusters().getClusters(), process.getClusters().getClusters()); - } - - /** - * Creates an empty process definition. - */ - public static ProcessMerlin getEmptyProcess(ProcessMerlin process) { - ProcessMerlin draft = new ProcessMerlin(process.toString()); - draft.setName(""); - draft.setTags(""); - draft.setACL(null); - draft.getInputs().getInputs().clear(); - draft.getOutputs().getOutputs().clear(); - draft.setRetry(null); - draft.clearProcessCluster(); - draft.clearProperties(); - draft.setFrequency(null); - draft.setOrder(null); - draft.setTimezone(null); - draft.setParallel(0); - Workflow workflow = new Workflow(); - workflow.setName(null); - workflow.setPath(null); - workflow.setVersion(null); - workflow.setEngine(null); - draft.setWorkflow(null, null, null); - return draft; - } - - /** - * Replaces old input by new input. - */ - public void resetInputFeed(String inputName, String feedName) { - Input in1 = getInputs().getInputs().get(0); - getInputs().getInputs().clear(); - Input in2 = new Input(); - in2.setEnd(in1.getEnd()); - in2.setFeed(feedName); - in2.setName(inputName); - in2.setPartition(in1.getPartition()); - in2.setStart(in1.getStart()); - in2.setOptional(in1.isOptional()); - getInputs().getInputs().add(in2); - } - - /** - * Replaces old output by new output. - */ - public void resetOutputFeed(String outputName, String feedName) { - Output out1 = getOutputs().getOutputs().get(0); - getOutputs().getOutputs().clear(); - Output out2 = new Output(); - out2.setFeed(feedName); - out2.setName(outputName); - out2.setInstance(out1.getInstance()); - getOutputs().getOutputs().add(out2); - } - - /** - * Adds array of feeds as input. - */ - public void addInputFeeds(String[] ipFeed) { - for(int i=0; i<ipFeed.length; i++){ - addInputFeed(ipFeed[i], ipFeed[i]); - } - } - - /** - * Adds array of feeds as output. - */ - public void addOutputFeeds(String[] opFeed) { - for(int i=0; i<opFeed.length; i++){ - addOutputFeed(opFeed[i], opFeed[i]); - } - } - -} - -
