http://git-wip-us.apache.org/repos/asf/falcon/blob/8e49379d/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/RecipeMerlin.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/RecipeMerlin.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/RecipeMerlin.java deleted file mode 100644 index 9b9cff2..0000000 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/RecipeMerlin.java +++ /dev/null @@ -1,366 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.falcon.regression.Entities; - -import org.apache.commons.configuration.AbstractFileConfiguration; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.filefilter.FalseFileFilter; -import org.apache.commons.io.filefilter.RegexFileFilter; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.falcon.cli.FalconCLI; -import org.apache.falcon.entity.v0.Frequency; -import org.apache.falcon.entity.v0.cluster.Interfacetype; -import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; -import org.apache.falcon.entity.v0.process.ACL; -import org.apache.falcon.entity.v0.process.PolicyType; -import org.apache.falcon.entity.v0.process.Retry; -import org.apache.falcon.regression.core.util.Config; -import org.apache.falcon.regression.core.util.OSUtil; -import org.apache.log4j.Logger; -import org.testng.Assert; - -import java.io.File; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.UUID; - -/** Class for representing a falcon recipe. */ -public final class RecipeMerlin { - private static final Logger LOGGER = Logger.getLogger(RecipeMerlin.class); - private static final String WORKFLOW_PATH_KEY = "falcon.recipe.workflow.path"; - private static final String RECIPE_NAME_KEY = "falcon.recipe.name"; - private static final String WRITE_DIR = - Config.getProperty("recipe.location", "/tmp/falcon-recipe"); - - private String template; - private AbstractFileConfiguration properties; - private String workflow; - private ClusterMerlin recipeCluster; - private ClusterMerlin srcCluster; - private ClusterMerlin tgtCluster; - - - public ClusterMerlin getRecipeCluster() { - return recipeCluster; - } - - public ClusterMerlin getSrcCluster() { - return srcCluster; - } - - public ClusterMerlin getTgtCluster() { - return tgtCluster; - } - - public FalconCLI.RecipeOperation getRecipeOperation() { - return recipeOperation; - } - - private FalconCLI.RecipeOperation recipeOperation; - - private RecipeMerlin() { - } - - public String getName() { - return properties.getString(RECIPE_NAME_KEY); - } - - public void setUniqueName(String prefix) { - properties.setProperty(RECIPE_NAME_KEY, prefix + UUID.randomUUID().toString().split("-")[0]); - } - - public String getSourceDir() { - return properties.getString("drSourceDir"); - } - - public RecipeMerlin withSourceDir(final String srcDir) { - properties.setProperty("drSourceDir", srcDir); - return this; - } - - public String getTargetDir() { - return properties.getString("drTargetDir"); - } - - public RecipeMerlin withTargetDir(final String tgtDir) { - properties.setProperty("drTargetDir", tgtDir); - return this; - } - - public String getSourceDb() { - return StringUtils.join(properties.getStringArray("sourceDatabase"), ','); - } - - public RecipeMerlin withSourceDb(final String srcDatabase) { - properties.setProperty("sourceDatabase", srcDatabase); - return this; - } - - public String getSourceTable() { - return StringUtils.join(properties.getStringArray("sourceTable"), ','); - } - - public RecipeMerlin withSourceTable(final String tgtTable) { - properties.setProperty("sourceTable", tgtTable); - return this; - } - - public RecipeMerlin withSourceCluster(ClusterMerlin sourceCluster) { - this.srcCluster = sourceCluster; - if (recipeOperation == FalconCLI.RecipeOperation.HDFS_REPLICATION) { - properties.setProperty("drSourceClusterFS", sourceCluster.getInterfaceEndpoint(Interfacetype.WRITE)); - } else { - properties.setProperty("sourceCluster", sourceCluster.getName()); - properties.setProperty("sourceMetastoreUri", sourceCluster.getProperty("hive.metastore.uris")); - properties.setProperty("sourceHiveServer2Uri", sourceCluster.getProperty("hive.server2.uri")); - //properties.setProperty("sourceServicePrincipal", - // sourceCluster.getProperty("hive.metastore.kerberos.principal")); - properties.setProperty("sourceStagingPath", sourceCluster.getLocation("staging")); - properties.setProperty("sourceNN", sourceCluster.getInterfaceEndpoint(Interfacetype.WRITE)); - properties.setProperty("sourceRM", sourceCluster.getInterfaceEndpoint(Interfacetype.EXECUTE)); - } - return this; - } - - public RecipeMerlin withTargetCluster(ClusterMerlin targetCluster) { - this.tgtCluster = targetCluster; - if (recipeOperation == FalconCLI.RecipeOperation.HDFS_REPLICATION) { - properties.setProperty("drTargetClusterFS", targetCluster.getInterfaceEndpoint(Interfacetype.WRITE)); - } else { - properties.setProperty("targetCluster", targetCluster.getName()); - properties.setProperty("targetMetastoreUri", targetCluster.getProperty("hive.metastore.uris")); - properties.setProperty("targetHiveServer2Uri", targetCluster.getProperty("hive.server2.uri")); - //properties.setProperty("targetServicePrincipal", - // targetCluster.getProperty("hive.metastore.kerberos.principal")); - properties.setProperty("targetStagingPath", targetCluster.getLocation("staging")); - properties.setProperty("targetNN", targetCluster.getInterfaceEndpoint(Interfacetype.WRITE)); - properties.setProperty("targetRM", targetCluster.getInterfaceEndpoint(Interfacetype.EXECUTE)); - } - return this; - } - - public RecipeMerlin withRecipeCluster(ClusterMerlin paramRecipeCluster) { - this.recipeCluster = paramRecipeCluster; - properties.setProperty("falcon.recipe.cluster.name", paramRecipeCluster.getName()); - properties.setProperty("falcon.recipe.cluster.hdfs.writeEndPoint", - paramRecipeCluster.getInterfaceEndpoint(Interfacetype.WRITE)); - return this; - } - - public RecipeMerlin withValidity(final String start, final String end) { - properties.setProperty("falcon.recipe.cluster.validity.start", start); - properties.setProperty("falcon.recipe.cluster.validity.end", end); - return this; - } - - public String getValidityStart() { - return properties.getString("falcon.recipe.cluster.validity.start"); - } - - public String getValidityEnd() { - return properties.getString("falcon.recipe.cluster.validity.end"); - } - - public RecipeMerlin withFrequency(final Frequency frequency) { - properties.setProperty("falcon.recipe.process.frequency", frequency.toString()); - return this; - } - - public Frequency getFrequency() { - return Frequency.fromString(properties.getString("falcon.recipe.process.frequency")); - } - - public String getMaxEvents() { - return properties.getString("maxEvents"); - } - - public String getReplicationMaxMaps() { - return properties.getString("replicationMaxMaps"); - } - - public String getDistCpMaxMaps() { - return properties.getString("distcpMaxMaps"); - } - - public String getMapBandwidth() { - return properties.getString("distcpMapBandwidth"); - } - - public Retry getRetry() { - final int retryAttempts = properties.getInt("falcon.recipe.retry.attempts"); - final String retryDelay = properties.getString("falcon.recipe.retry.delay"); - final String retryPolicy = properties.getString("falcon.recipe.retry.policy"); - - Retry retry = new Retry(); - retry.setAttempts(retryAttempts); - retry.setDelay(Frequency.fromString(retryDelay)); - retry.setPolicy(PolicyType.fromValue(retryPolicy)); - return retry; - } - - public ACL getAcl() { - ACL acl = new ACL(); - acl.setOwner(properties.getString("falcon.recipe.acl.owner")); - acl.setGroup(properties.getString("falcon.recipe.acl.group")); - acl.setPermission(properties.getString("falcon.recipe.acl.permission")); - return acl; - } - - - /** - * Read recipe from a given directory. Expecting that recipe will follow these conventions. - * <br> 1. properties file will have .properties extension - * <br> 2. template file will have end with -template.xml - * <br> 3. workflow file will have end with -workflow.xml - * @param readPath the location from where recipe will be read - * @param recipeOperation operation of this recipe - */ - public static RecipeMerlin readFromDir(final String readPath, - FalconCLI.RecipeOperation recipeOperation) { - Assert.assertTrue(StringUtils.isNotEmpty(readPath), "readPath for recipe can't be empty"); - Assert.assertNotNull(recipeOperation, "readPath for recipe can't be empty"); - RecipeMerlin instance = new RecipeMerlin(); - instance.recipeOperation = recipeOperation; - LOGGER.info("Loading recipe from directory: " + readPath); - File directory = null; - try { - directory = new File(RecipeMerlin.class.getResource("/" + readPath).toURI()); - } catch (URISyntaxException e) { - Assert.fail("could not find dir: " + readPath); - } - final Collection<File> propertiesFiles = FileUtils.listFiles(directory, - new RegexFileFilter(".*\\.properties"), FalseFileFilter.INSTANCE); - Assert.assertEquals(propertiesFiles.size(), 1, - "Expecting only one property file at: " + readPath +" found: " + propertiesFiles); - try { - instance.properties = - new PropertiesConfiguration(propertiesFiles.iterator().next()); - } catch (ConfigurationException e) { - Assert.fail("Couldn't read recipe's properties file because of exception: " - + ExceptionUtils.getStackTrace(e)); - } - instance.properties.setFileName(null); //prevent accidental overwrite of template - //removing defaults - specific test need to supplied this - instance.properties.clearProperty("sourceDatabase"); - instance.properties.clearProperty("sourceTable"); - instance.properties.clearProperty("targetDatabase"); - instance.properties.clearProperty("targetTable"); - instance.properties.setProperty("falcon.recipe.acl.owner", MerlinConstants.CURRENT_USER_NAME); - instance.properties.setProperty("falcon.recipe.acl.group", MerlinConstants.CURRENT_USER_GROUP); - instance.properties.setProperty("falcon.recipe.acl.permission", "*"); - - final Collection<File> templatesFiles = FileUtils.listFiles(directory, - new RegexFileFilter(".*-template\\.xml"), FalseFileFilter.INSTANCE); - Assert.assertEquals(templatesFiles.size(), 1, - "Expecting only one template file at: " + readPath + " found: " + templatesFiles); - try { - instance.template = - FileUtils.readFileToString(templatesFiles.iterator().next()); - } catch (IOException e) { - Assert.fail("Couldn't read recipe's template file because of exception: " - + ExceptionUtils.getStackTrace(e)); - } - - final Collection<File> workflowFiles = FileUtils.listFiles(directory, - new RegexFileFilter(".*-workflow\\.xml"), FalseFileFilter.INSTANCE); - Assert.assertEquals(workflowFiles.size(), 1, - "Expecting only one workflow file at: " + readPath + " found: " + workflowFiles); - try { - instance.workflow = FileUtils.readFileToString(workflowFiles.iterator().next()); - } catch (IOException e) { - Assert.fail("Couldn't read recipe's workflow file because of exception: " - + ExceptionUtils.getStackTrace(e)); - } - return instance; - } - - /** - * Write recipe. - */ - private void write() { - final String templateFileLocation = OSUtil.concat(WRITE_DIR, getName() + "-template.xml"); - try { - Assert.assertNotNull(templateFileLocation, - "Write location for template file is unexpectedly null."); - FileUtils.writeStringToFile(new File(templateFileLocation), template); - } catch (IOException e) { - Assert.fail("Couldn't write recipe's template file because of exception: " - + ExceptionUtils.getStackTrace(e)); - } - - final String workflowFileLocation = OSUtil.concat(WRITE_DIR, getName() + "-workflow.xml"); - try { - Assert.assertNotNull(workflowFileLocation, - "Write location for workflow file is unexpectedly null."); - FileUtils.writeStringToFile(new File(workflowFileLocation), workflow); - } catch (IOException e) { - Assert.fail("Couldn't write recipe's workflow file because of exception: " - + ExceptionUtils.getStackTrace(e)); - } - properties.setProperty(WORKFLOW_PATH_KEY, workflowFileLocation); - properties.setProperty("falcon.recipe.workflow.name", getName() + "-workflow"); - - final String propFileLocation = OSUtil.concat(WRITE_DIR, getName() + ".properties"); - try { - Assert.assertNotNull(propFileLocation, - "Write location for properties file is unexpectedly null."); - properties.save(new File(propFileLocation)); - } catch (ConfigurationException e) { - Assert.fail("Couldn't write recipe's process file because of exception: " - + ExceptionUtils.getStackTrace(e)); - } - } - - /** - * Get submission command. - */ - public List<String> getSubmissionCommand() { - write(); - final List<String> cmd = new ArrayList<>(); - Collections.addAll(cmd, "recipe", "-name", getName(), - "-operation", recipeOperation.toString()); - return cmd; - } - - /** - * Set tags for recipe. - */ - public List<String> getTags() { - final String tagsStr = properties.getString("falcon.recipe.tags"); - if (StringUtils.isEmpty(tagsStr)) { - return new ArrayList<>(); - } - return Arrays.asList(tagsStr.split(",")); - } - - /** - * Set tags for recipe. - */ - public void setTags(List<String> tags) { - properties.setProperty("falcon.recipe.tags", StringUtils.join(tags, ',')); - } -}
http://git-wip-us.apache.org/repos/asf/falcon/blob/8e49379d/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/TestEntityUtil.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/TestEntityUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/TestEntityUtil.java deleted file mode 100644 index a9a8f00..0000000 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/Entities/TestEntityUtil.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.falcon.regression.Entities; - -import org.apache.falcon.entity.v0.Entity; -import org.apache.falcon.entity.v0.EntityType; -import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; -import org.apache.log4j.Logger; -import org.testng.Assert; - -import javax.xml.bind.Unmarshaller; -import java.io.StringReader; -import java.util.UUID; - -/** - * Util class for merlin entities. - */ -final class TestEntityUtil { - - private static final Logger LOGGER = Logger.getLogger(TestEntityUtil.class); - private TestEntityUtil() { - throw new AssertionError("Instantiating utility class..."); - } - - public static Entity fromString(EntityType type, String str) { - try { - Unmarshaller unmarshaller = type.getUnmarshaller(); - unmarshaller.setSchema(null); - return (Entity) unmarshaller.unmarshal(new StringReader(str)); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - /* - Deprecating entity name if is_deprecate is set. - Useful when oozie uses embedded database(derby) - */ - public static String generateUniqueName(String prefix, String oldName) { - Assert.assertNotNull(prefix, "name prefix shouldn't be null!"); - String randomPart = UUID.randomUUID().toString().split("-")[0]; - if (MerlinConstants.IS_DEPRECATE) { - return 'A' + Integer.toHexString(prefix.hashCode()) + '-' + randomPart; - } else { - return prefix + '-' + oldName + '-' + randomPart; - } - } -} http://git-wip-us.apache.org/repos/asf/falcon/blob/8e49379d/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java deleted file mode 100644 index 9758d12..0000000 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/bundle/Bundle.java +++ /dev/null @@ -1,927 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.falcon.regression.core.bundle; - -import org.apache.commons.lang.StringUtils; -import org.apache.falcon.cli.FalconCLI; -import org.apache.falcon.entity.v0.Frequency; -import org.apache.falcon.entity.v0.Frequency.TimeUnit; -import org.apache.falcon.entity.v0.cluster.Interfacetype; -import org.apache.falcon.entity.v0.feed.ClusterType; -import org.apache.falcon.entity.v0.feed.Feed; -import org.apache.falcon.entity.v0.process.Cluster; -import org.apache.falcon.entity.v0.process.EngineType; -import org.apache.falcon.entity.v0.process.LateProcess; -import org.apache.falcon.entity.v0.process.Retry; -import org.apache.falcon.regression.Entities.ClusterMerlin; -import org.apache.falcon.regression.Entities.FeedMerlin; -import org.apache.falcon.regression.Entities.ProcessMerlin; -import org.apache.falcon.regression.core.helpers.ColoHelper; -import org.apache.falcon.regression.core.response.ServiceResponse; -import org.apache.falcon.regression.core.util.AssertUtil; -import org.apache.falcon.regression.core.util.BundleUtil; -import org.apache.falcon.regression.core.util.TimeUtil; -import org.apache.falcon.regression.core.util.Util; -import org.apache.hadoop.security.authentication.client.AuthenticationException; -import org.apache.log4j.Logger; -import org.testng.Assert; - -import javax.xml.bind.JAXBException; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * A bundle abstraction. - */ -public class Bundle { - - private static final String PRISM_PREFIX = "prism"; - private static ColoHelper prismHelper = new ColoHelper(PRISM_PREFIX); - private static final Logger LOGGER = Logger.getLogger(Bundle.class); - - private List<String> clusters; - private List<String> dataSets; - private String processData; - - public void submitFeed() - throws URISyntaxException, IOException, AuthenticationException, JAXBException, - InterruptedException { - submitClusters(prismHelper); - - AssertUtil.assertSucceeded(prismHelper.getFeedHelper().submitEntity(dataSets.get(0))); - } - - public void submitAndScheduleFeed() throws Exception { - submitClusters(prismHelper); - - AssertUtil.assertSucceeded(prismHelper.getFeedHelper().submitAndSchedule(dataSets.get(0))); - } - - public void submitAndScheduleFeedUsingColoHelper(ColoHelper coloHelper) throws Exception { - submitFeed(); - - AssertUtil.assertSucceeded(coloHelper.getFeedHelper().schedule(dataSets.get(0))); - } - - public void submitAndScheduleAllFeeds() - throws JAXBException, IOException, URISyntaxException, AuthenticationException, - InterruptedException { - submitClusters(prismHelper); - - for (String feed : dataSets) { - AssertUtil.assertSucceeded(prismHelper.getFeedHelper().submitAndSchedule(feed)); - } - } - - public ServiceResponse submitProcess(boolean shouldSucceed) throws JAXBException, - IOException, URISyntaxException, AuthenticationException, InterruptedException { - submitClusters(prismHelper); - submitFeeds(prismHelper); - ServiceResponse r = prismHelper.getProcessHelper().submitEntity(processData); - if (shouldSucceed) { - AssertUtil.assertSucceeded(r); - } else { - AssertUtil.assertFailed(r); - } - return r; - } - - public void submitFeedsScheduleProcess() throws Exception { - submitClusters(prismHelper); - - submitFeeds(prismHelper); - - AssertUtil.assertSucceeded(prismHelper.getProcessHelper().submitAndSchedule(processData)); - } - - - public void submitAndScheduleProcess() throws Exception { - submitAndScheduleAllFeeds(); - - AssertUtil.assertSucceeded(prismHelper.getProcessHelper().submitAndSchedule(processData)); - } - - public void submitAndScheduleProcessUsingColoHelper(ColoHelper coloHelper) throws Exception { - submitProcess(true); - - AssertUtil.assertSucceeded(coloHelper.getProcessHelper().schedule(processData)); - } - - public List<String> getClusters() { - return clusters; - } - - public Bundle(String clusterData, List<String> dataSets, String processData) { - this.dataSets = dataSets; - this.processData = processData; - this.clusters = new ArrayList<>(); - this.clusters.add(clusterData); - } - - public Bundle(Bundle bundle, String prefix) { - this.dataSets = new ArrayList<>(bundle.getDataSets()); - this.processData = bundle.getProcessData(); - this.clusters = new ArrayList<>(); - for (String cluster : bundle.getClusters()) { - this.clusters.add(BundleUtil.getEnvClusterXML(cluster, prefix).toString()); - } - } - - public Bundle(Bundle bundle, ColoHelper helper) { - this(bundle, helper.getPrefix()); - } - - public void setClusterData(List<String> pClusters) { - this.clusters = new ArrayList<>(pClusters); - } - /** - * Unwraps cluster element to string and writes it to bundle. - * - * @param c Cluster object to be unwrapped and set into bundle - */ - public void writeClusterElement(org.apache.falcon.entity.v0.cluster.Cluster c) { - final List<String> newClusters = new ArrayList<>(); - newClusters.add(c.toString()); - setClusterData(newClusters); - } - - /** - * Wraps bundle cluster in a Cluster object. - * - * @return cluster definition in a form of Cluster object - */ - public ClusterMerlin getClusterElement() { - return new ClusterMerlin(getClusters().get(0)); - } - - - public List<String> getClusterNames() { - List<String> clusterNames = new ArrayList<>(); - for (String cluster : clusters) { - clusterNames.add(new ClusterMerlin(cluster).getName()); - } - return clusterNames; - } - - public List<String> getDataSets() { - return dataSets; - } - - public void setDataSets(List<String> dataSets) { - this.dataSets = dataSets; - } - - public String getProcessData() { - return processData; - } - - public void setProcessData(String processData) { - this.processData = processData; - } - - /** - * Generates unique entities within a bundle changing their names and names of dependant items - * to unique. - */ - public void generateUniqueBundle(Object testClassObject) { - generateUniqueBundle(testClassObject.getClass().getSimpleName()); - } - - /** - * Generates unique entities within a bundle changing their names and names of dependant items - * to unique. - */ - public void generateUniqueBundle(String prefix) { - /* creating new names */ - List<ClusterMerlin> clusterMerlinList = BundleUtil.getClustersFromStrings(clusters); - Map<String, String> clusterNameMap = new HashMap<>(); - for (ClusterMerlin clusterMerlin : clusterMerlinList) { - clusterNameMap.putAll(clusterMerlin.setUniqueName(prefix)); - } - - List<FeedMerlin> feedMerlinList = FeedMerlin.fromString(dataSets); - Map<String, String> feedNameMap = new HashMap<>(); - for (FeedMerlin feedMerlin : feedMerlinList) { - feedNameMap.putAll(feedMerlin.setUniqueName(prefix)); - } - - /* setting new names in feeds and process */ - for (FeedMerlin feedMerlin : feedMerlinList) { - feedMerlin.renameClusters(clusterNameMap); - } - - /* setting variables */ - clusters.clear(); - for (ClusterMerlin clusterMerlin : clusterMerlinList) { - clusters.add(clusterMerlin.toString()); - } - dataSets.clear(); - for (FeedMerlin feedMerlin : feedMerlinList) { - dataSets.add(feedMerlin.toString()); - } - - if (StringUtils.isNotEmpty(processData)) { - ProcessMerlin processMerlin = new ProcessMerlin(processData); - processMerlin.setUniqueName(prefix); - processMerlin.renameClusters(clusterNameMap); - processMerlin.renameFeeds(feedNameMap); - processData = processMerlin.toString(); - } - } - - public ServiceResponse submitBundle(ColoHelper helper) - throws JAXBException, IOException, URISyntaxException, AuthenticationException, - InterruptedException { - - submitClusters(helper); - - //lets submit all data first - submitFeeds(helper); - - return helper.getProcessHelper().submitEntity(getProcessData()); - } - - /** - * Submit all the entities and schedule the process. - * - * @param helper helper of prism host - * @return schedule response or cluster submit response if it fails - * @throws IOException - * @throws JAXBException - * @throws URISyntaxException - * @throws AuthenticationException - */ - public ServiceResponse submitFeedsScheduleProcess(ColoHelper helper) - throws IOException, JAXBException, URISyntaxException, - AuthenticationException, InterruptedException { - ServiceResponse submitResponse = submitBundle(helper); - if (submitResponse.getCode() == 400) { - return submitResponse; - } - - //lets schedule the damn thing now :) - ServiceResponse scheduleResult = helper.getProcessHelper().schedule(getProcessData()); - AssertUtil.assertSucceeded(scheduleResult); - TimeUtil.sleepSeconds(7); - return scheduleResult; - } - - /** - * Sets the only process input. - * - * @param startEl its start in terms of EL expression - * @param endEl its end in terms of EL expression - */ - public void setProcessInput(String startEl, String endEl) { - ProcessMerlin process = getProcessObject(); - process.setInputFeedWithEl(Util.readEntityName(getInputFeedFromBundle()), startEl, endEl); - this.setProcessData(process.toString()); - } - - public void setInvalidData() { - FeedMerlin dataElement = new FeedMerlin(getInputFeedFromBundle()); - String oldLocation = dataElement.getLocations().getLocations().get(0).getPath(); - LOGGER.info("oldlocation: " + oldLocation); - dataElement.getLocations().getLocations().get(0).setPath( - oldLocation.substring(0, oldLocation.indexOf('$')) + "invalid/" - + oldLocation.substring(oldLocation.indexOf('$'))); - LOGGER.info("new location: " + dataElement.getLocations().getLocations().get(0).getPath()); - setInputFeed(dataElement.toString()); - } - - public void setInputFeed(String newFeed) { - String inputFeedName = getInputFeedNameFromBundle(); - for (int i = 0; i < dataSets.size(); i++) { - if (new FeedMerlin(dataSets.get(i)).getName().equals(inputFeedName)) { - dataSets.set(i, newFeed); - return; - } - } - } - - public void setFeedValidity(String feedStart, String feedEnd, String feedName) { - FeedMerlin feedElement = getFeedElement(feedName); - feedElement.setValidity(feedStart, feedEnd); - writeFeedElement(feedElement, feedName); - } - - public int getInitialDatasetFrequency() { - FeedMerlin dataElement = new FeedMerlin(getInputFeedFromBundle()); - if (dataElement.getFrequency().getTimeUnit() == TimeUnit.hours) { - return (Integer.parseInt(dataElement.getFrequency().getFrequency())) * 60; - } else { - return (Integer.parseInt(dataElement.getFrequency().getFrequency())); - } - } - - public Date getStartInstanceProcess(Calendar time) { - ProcessMerlin processElement = getProcessObject(); - LOGGER.info("start instance: " + processElement.getInputs().getInputs().get(0).getStart()); - return TimeUtil.getMinutes(processElement.getInputs().getInputs().get(0).getStart(), time); - } - - public Date getEndInstanceProcess(Calendar time) { - ProcessMerlin processElement = getProcessObject(); - LOGGER.info("end instance: " + processElement.getInputs().getInputs().get(0).getEnd()); - LOGGER.info("timezone in getendinstance: " + time.getTimeZone().toString()); - LOGGER.info("time in getendinstance: " + time.getTime()); - return TimeUtil.getMinutes(processElement.getInputs().getInputs().get(0).getEnd(), time); - } - - public void setDatasetInstances(String startInstance, String endInstance) { - ProcessMerlin processElement = getProcessObject(); - processElement.setDatasetInstances(startInstance, endInstance); - setProcessData(processElement.toString()); - } - - public void setProcessPeriodicity(int frequency, TimeUnit periodicity) { - ProcessMerlin processElement = getProcessObject(); - processElement.setPeriodicity(frequency, periodicity); - setProcessData(processElement.toString()); - } - - public void setProcessInputStartEnd(String start, String end) { - ProcessMerlin processElement = getProcessObject(); - processElement.setProcessInputStartEnd(start, end); - setProcessData(processElement.toString()); - } - - public void setOutputFeedPeriodicity(int frequency, TimeUnit periodicity) { - ProcessMerlin processElement = new ProcessMerlin(processData); - String outputDataset = null; - int datasetIndex; - for (datasetIndex = 0; datasetIndex < dataSets.size(); datasetIndex++) { - outputDataset = dataSets.get(datasetIndex); - if (outputDataset.contains(processElement.getOutputs().getOutputs().get(0).getFeed())) { - break; - } - } - - FeedMerlin feedElement = new FeedMerlin(outputDataset); - - feedElement.setFrequency(new Frequency("" + frequency, periodicity)); - dataSets.set(datasetIndex, feedElement.toString()); - LOGGER.info("modified o/p dataSet is: " + dataSets.get(datasetIndex)); - } - - public int getProcessConcurrency() { - return getProcessObject().getParallel(); - } - - public void setOutputFeedLocationData(String path) { - FeedMerlin feedElement = getFeedElement(getOutputFeedNameFromBundle()); - feedElement.setDataLocationPath(path); - writeFeedElement(feedElement, feedElement.getName()); - LOGGER.info("modified location path dataSet is: " + feedElement); - } - - public void setProcessConcurrency(int concurrency) { - ProcessMerlin processElement = getProcessObject(); - processElement.setParallel((concurrency)); - setProcessData(processElement.toString()); - } - - public void setProcessWorkflow(String wfPath) { - setProcessWorkflow(wfPath, null); - } - - public void setProcessWorkflow(String wfPath, EngineType engineType) { - setProcessWorkflow(wfPath, null, engineType); - } - - public void setProcessWorkflow(String wfPath, String libPath, EngineType engineType) { - ProcessMerlin processElement = getProcessObject(); - processElement.setWorkflow(wfPath, libPath, engineType); - setProcessData(processElement.toString()); - } - - public ProcessMerlin getProcessObject() { - return new ProcessMerlin(getProcessData()); - } - - public FeedMerlin getFeedElement(String feedName) { - return new FeedMerlin(getFeed(feedName)); - } - - - public String getFeed(String feedName) { - for (String feed : getDataSets()) { - if (Util.readEntityName(feed).contains(feedName)) { - return feed; - } - } - - return null; - } - - - public void writeFeedElement(FeedMerlin feedElement, String feedName) { - writeFeedElement(feedElement.toString(), feedName); - } - - - public void writeFeedElement(String feedString, String feedName) { - dataSets.set(dataSets.indexOf(getFeed(feedName)), feedString); - } - - - public void setInputFeedPeriodicity(int frequency, TimeUnit periodicity) { - String feedName = getInputFeedNameFromBundle(); - FeedMerlin feedElement = getFeedElement(feedName); - feedElement.setPeriodicity(frequency, periodicity); - writeFeedElement(feedElement, feedName); - - } - - public void setInputFeedValidity(String startInstance, String endInstance) { - String feedName = getInputFeedNameFromBundle(); - this.setFeedValidity(startInstance, endInstance, feedName); - } - - public void setOutputFeedValidity(String startInstance, String endInstance) { - String feedName = getOutputFeedNameFromBundle(); - this.setFeedValidity(startInstance, endInstance, feedName); - } - - public void setInputFeedDataPath(String path) { - String feedName = getInputFeedNameFromBundle(); - FeedMerlin feedElement = getFeedElement(feedName); - feedElement.setDataLocationPath(path); - writeFeedElement(feedElement, feedName); - } - - public String getFeedDataPathPrefix() { - FeedMerlin feedElement = - getFeedElement(getInputFeedNameFromBundle()); - return Util.getPathPrefix(feedElement.getLocations().getLocations().get(0) - .getPath()); - } - - public void setProcessValidity(String startDate, String endDate) { - ProcessMerlin processElement = new ProcessMerlin(processData); - processElement.setValidity(startDate, endDate); - processData = processElement.toString(); - } - - public void setProcessLatePolicy(LateProcess lateProcess) { - ProcessMerlin processElement = new ProcessMerlin(processData); - processElement.setLateProcess(lateProcess); - processData = processElement.toString(); - } - - - public void verifyDependencyListing(ColoHelper coloHelper) - throws InterruptedException, IOException, AuthenticationException, URISyntaxException { - //display dependencies of process: - String dependencies = coloHelper.getProcessHelper().getDependencies( - Util.readEntityName(getProcessData())).getEntityList().toString(); - - //verify presence - for (String cluster : clusters) { - Assert.assertTrue(dependencies.contains("(cluster) " + Util.readEntityName(cluster))); - } - for (String feed : getDataSets()) { - Assert.assertTrue(dependencies.contains("(feed) " + Util.readEntityName(feed))); - for (String cluster : clusters) { - Assert.assertTrue(coloHelper.getFeedHelper().getDependencies( - Util.readEntityName(feed)).getEntityList().toString() - .contains("(cluster) " + Util.readEntityName(cluster))); - } - Assert.assertFalse(coloHelper.getFeedHelper().getDependencies( - Util.readEntityName(feed)).getEntityList().toString() - .contains("(process)" + Util.readEntityName(getProcessData()))); - } - } - - public void addProcessInput(String inputName, String feedName) { - ProcessMerlin processElement = getProcessObject(); - processElement.addInputFeed(inputName, feedName); - setProcessData(processElement.toString()); - } - - public void setProcessName(String newName) { - ProcessMerlin processElement = getProcessObject(); - processElement.setName(newName); - setProcessData(processElement.toString()); - - } - - public void setRetry(Retry retry) { - LOGGER.info("old process: " + Util.prettyPrintXml(processData)); - ProcessMerlin processObject = getProcessObject(); - processObject.setRetry(retry); - processData = processObject.toString(); - LOGGER.info("updated process: " + Util.prettyPrintXml(processData)); - } - - public void setInputFeedAvailabilityFlag(String flag) { - String feedName = getInputFeedNameFromBundle(); - FeedMerlin feedElement = getFeedElement(feedName); - feedElement.setAvailabilityFlag(flag); - writeFeedElement(feedElement, feedName); - } - - public void setOutputFeedAvailabilityFlag(String flag) { - String feedName = getOutputFeedNameFromBundle(); - FeedMerlin feedElement = getFeedElement(feedName); - feedElement.setAvailabilityFlag(flag); - writeFeedElement(feedElement, feedName); - } - - public void setCLusterColo(String colo) { - ClusterMerlin c = getClusterElement(); - c.setColo(colo); - writeClusterElement(c); - - } - - public void setClusterInterface(Interfacetype interfacetype, String value) { - ClusterMerlin c = getClusterElement(); - c.setInterface(interfacetype, value, null); - writeClusterElement(c); - } - - public void setInputFeedTableUri(String tableUri) { - final String feedStr = getInputFeedFromBundle(); - FeedMerlin feed = new FeedMerlin(feedStr); - feed.setTableUri(tableUri); - writeFeedElement(feed, feed.getName()); - } - - public void setOutputFeedTableUri(String tableUri) { - final String feedStr = getOutputFeedFromBundle(); - FeedMerlin feed = new FeedMerlin(feedStr); - feed.setTableUri(tableUri); - writeFeedElement(feed, feed.getName()); - } - - public void setCLusterWorkingPath(String clusterData, String path) { - ClusterMerlin c = new ClusterMerlin(clusterData); - c.setWorkingLocationPath(path); - writeClusterElement(c); - } - - - public void submitClusters(ColoHelper helper) - throws JAXBException, IOException, URISyntaxException, AuthenticationException, - InterruptedException { - submitClusters(helper, null); - } - - public void submitClusters(ColoHelper helper, String user) - throws JAXBException, IOException, URISyntaxException, AuthenticationException, - InterruptedException { - for (String cluster : this.clusters) { - AssertUtil.assertSucceeded(helper.getClusterHelper().submitEntity(cluster, user)); - } - } - - public void submitFeeds(ColoHelper helper) - throws JAXBException, IOException, URISyntaxException, AuthenticationException, - InterruptedException { - for (String feed : this.dataSets) { - AssertUtil.assertSucceeded(helper.getFeedHelper().submitEntity(feed)); - } - } - - public void addClusterToBundle(String clusterData, ClusterType type, - String startTime, String endTime) { - clusterData = setNewClusterName(clusterData); - - this.clusters.add(clusterData); - //now to add clusters to feeds - for (int i = 0; i < dataSets.size(); i++) { - FeedMerlin feedObject = new FeedMerlin(dataSets.get(i)); - org.apache.falcon.entity.v0.feed.Cluster cluster = - new org.apache.falcon.entity.v0.feed.Cluster(); - cluster.setName(new ClusterMerlin(clusterData).getName()); - cluster.setValidity(feedObject.getClusters().getClusters().get(0).getValidity()); - cluster.setType(type); - cluster.setRetention(feedObject.getClusters().getClusters().get(0).getRetention()); - feedObject.getClusters().getClusters().add(cluster); - - dataSets.remove(i); - dataSets.add(i, feedObject.toString()); - - } - - //now to add cluster to process - ProcessMerlin processObject = new ProcessMerlin(processData); - Cluster cluster = new Cluster(); - cluster.setName(new ClusterMerlin(clusterData).getName()); - org.apache.falcon.entity.v0.process.Validity v = - processObject.getClusters().getClusters().get(0).getValidity(); - if (StringUtils.isNotEmpty(startTime)) { - v.setStart(TimeUtil.oozieDateToDate(startTime).toDate()); - } - if (StringUtils.isNotEmpty(endTime)) { - v.setEnd(TimeUtil.oozieDateToDate(endTime).toDate()); - } - cluster.setValidity(v); - processObject.getClusters().getClusters().add(cluster); - this.processData = processObject.toString(); - - } - - private String setNewClusterName(String clusterData) { - ClusterMerlin clusterObj = new ClusterMerlin(clusterData); - clusterObj.setName(clusterObj.getName() + this.clusters.size() + 1); - return clusterObj.toString(); - } - - public void deleteBundle(ColoHelper helper) { - - try { - helper.getProcessHelper().delete(getProcessData()); - } catch (Exception e) { - e.getStackTrace(); - } - - for (String dataset : getDataSets()) { - try { - helper.getFeedHelper().delete(dataset); - } catch (Exception e) { - e.getStackTrace(); - } - } - - for (String cluster : this.getClusters()) { - try { - helper.getClusterHelper().delete(cluster); - } catch (Exception e) { - e.getStackTrace(); - } - } - - - } - - public String getProcessName() { - return new ProcessMerlin(this.processData).getName(); - } - - public void setProcessLibPath(String libPath) { - ProcessMerlin processElement = getProcessObject(); - processElement.getWorkflow().setLib(libPath); - setProcessData(processElement.toString()); - } - - public void setProcessTimeOut(int magnitude, TimeUnit unit) { - ProcessMerlin processElement = getProcessObject(); - processElement.setTimeOut(magnitude, unit); - setProcessData(processElement.toString()); - } - - public static void submitCluster(Bundle... bundles) - throws IOException, URISyntaxException, AuthenticationException, InterruptedException { - - for (Bundle bundle : bundles) { - ServiceResponse r = - prismHelper.getClusterHelper().submitEntity(bundle.getClusters().get(0)); - Assert.assertTrue(r.getMessage().contains("SUCCEEDED"), r.getMessage()); - } - - - } - - /** - * Generates unique entities definitions: clusters, feeds and process, populating them with - * desired values of different properties. - * - * @param numberOfClusters number of clusters on which feeds and process should run - * @param numberOfInputs number of desired inputs in process definition - * @param numberOfOptionalInput how many inputs should be optional - * @param inputBasePaths base data path for inputs - * @param numberOfOutputs number of outputs - * @param startTime start of feeds and process validity on every cluster - * @param endTime end of feeds and process validity on every cluster - */ - public void generateRequiredBundle(int numberOfClusters, int numberOfInputs, - int numberOfOptionalInput, - String inputBasePaths, int numberOfOutputs, String startTime, - String endTime) { - //generate and set clusters - ClusterMerlin c = new ClusterMerlin(getClusters().get(0)); - List<String> newClusters = new ArrayList<>(); - final String clusterName = c.getName(); - for (int i = 0; i < numberOfClusters; i++) { - c.setName(clusterName + i); - newClusters.add(i, c.toString()); - } - setClusterData(newClusters); - - //generate and set newDataSets - List<String> newDataSets = new ArrayList<>(); - for (int i = 0; i < numberOfInputs; i++) { - final FeedMerlin feed = new FeedMerlin(getDataSets().get(0)); - feed.setName(feed.getName() + "-input" + i); - feed.setFeedClusters(newClusters, inputBasePaths + "/input" + i, startTime, endTime); - newDataSets.add(feed.toString()); - } - for (int i = 0; i < numberOfOutputs; i++) { - final FeedMerlin feed = new FeedMerlin(getDataSets().get(0)); - feed.setName(feed.getName() + "-output" + i); - feed.setFeedClusters(newClusters, inputBasePaths + "/output" + i, startTime, endTime); - newDataSets.add(feed.toString()); - } - setDataSets(newDataSets); - - //add clusters and feed to process - ProcessMerlin processMerlin = new ProcessMerlin(getProcessData()); - processMerlin.setProcessClusters(newClusters, startTime, endTime); - processMerlin.setProcessFeeds(newDataSets, numberOfInputs, - numberOfOptionalInput, numberOfOutputs); - setProcessData(processMerlin.toString()); - } - - public void submitAndScheduleBundle(ColoHelper helper, boolean checkSuccess) - throws IOException, JAXBException, URISyntaxException, AuthenticationException, - InterruptedException { - - for (int i = 0; i < getClusters().size(); i++) { - ServiceResponse r; - r = helper.getClusterHelper().submitEntity(getClusters().get(i)); - if (checkSuccess) { - AssertUtil.assertSucceeded(r); - } - } - for (int i = 0; i < getDataSets().size(); i++) { - ServiceResponse r = helper.getFeedHelper().submitAndSchedule(getDataSets().get(i)); - if (checkSuccess) { - AssertUtil.assertSucceeded(r); - } - } - ServiceResponse r = helper.getProcessHelper().submitAndSchedule(getProcessData()); - if (checkSuccess) { - AssertUtil.assertSucceeded(r); - } - } - - /** - * Changes names of process inputs. - * - * @param names desired names of inputs - */ - public void setProcessInputNames(String... names) { - ProcessMerlin p = new ProcessMerlin(processData); - p.setInputNames(names); - processData = p.toString(); - } - - /** - * Adds optional property to process definition. - * - * @param properties desired properties to be added - */ - public void addProcessProperty(String propName, String propValue) { - processData = new ProcessMerlin(processData).withProperty(propName, propValue).toString(); - } - - /** - * Sets partition for each input, according to number of supplied partitions. - * - * @param partition partitions to be set - */ - public void setProcessInputPartition(String... partition) { - ProcessMerlin p = new ProcessMerlin(processData); - p.setInputPartition(partition); - processData = p.toString(); - } - - /** - * Sets name(s) of the process output(s). - * - * @param names new names of the outputs - */ - public void setProcessOutputNames(String... names) { - ProcessMerlin p = new ProcessMerlin(processData); - p.setOutputNames(names); - processData = p.toString(); - } - - public void addInputFeedToBundle(String feedRefName, Feed feed) { - this.getDataSets().add(feed.toString()); - - ProcessMerlin processObject = new ProcessMerlin(processData); - processObject.addInputFeed(feedRefName, feed.getName()); - setProcessData(processObject.toString()); - } - - public void addOutputFeedToBundle(String feedRefName, Feed feed) { - this.getDataSets().add(feed.toString()); - - ProcessMerlin processObject = getProcessObject(); - processObject.addOutputFeed(feedRefName, feed.getName()); - setProcessData(processObject.toString()); - } - - public void setProcessProperty(String property, String value) { - ProcessMerlin process = getProcessObject().withProperty(property, value); - this.setProcessData(process.toString()); - } - - public String getDatasetPath() { - FeedMerlin dataElement = new FeedMerlin(getInputFeedFromBundle()); - return dataElement.getLocations().getLocations().get(0).getPath(); - } - - public String getInputFeedFromBundle() { - ProcessMerlin processObject = new ProcessMerlin(getProcessData()); - return getFeed(processObject.getInputs().getInputs().get(0).getFeed()); - } - - public String getOutputFeedFromBundle() { - ProcessMerlin processObject = new ProcessMerlin(getProcessData()); - return getFeed(processObject.getOutputs().getOutputs().get(0).getFeed()); - } - - public String getOutputFeedNameFromBundle() { - String feedData = getOutputFeedFromBundle(); - FeedMerlin feedObject = new FeedMerlin(feedData); - return feedObject.getName(); - } - - public String getInputFeedNameFromBundle() { - String feedData = getInputFeedFromBundle(); - FeedMerlin feedObject = new FeedMerlin(feedData); - return feedObject.getName(); - } - - /** - * Sets process pipelines. - * @param pipelines proposed pipelines - */ - public void setProcessPipeline(String... pipelines){ - ProcessMerlin process = new ProcessMerlin(getProcessData()); - process.setPipelineTag(pipelines); - setProcessData(process.toString()); - } - - /** - * Set ACL of bundle's cluster. - */ - public void setCLusterACL(String owner, String group, String permission) { - ClusterMerlin clusterMerlin = getClusterElement(); - clusterMerlin.setACL(owner, group, permission); - writeClusterElement(clusterMerlin); - - } - - /** - * Set ACL of bundle's input feed. - */ - public void setInputFeedACL(String owner, String group, String permission) { - String feedName = getInputFeedNameFromBundle(); - FeedMerlin feedMerlin = getFeedElement(feedName); - feedMerlin.setACL(owner, group, permission); - writeFeedElement(feedMerlin, feedName); - } - - /** - * Set ACL of bundle's process. - */ - public void setProcessACL(String owner, String group, String permission) { - ProcessMerlin processMerlin = getProcessObject(); - processMerlin.setACL(owner, group, permission); - setProcessData(processMerlin.toString()); - } - - /** - * Set custom tags for a process. Key-value pairs are valid. - */ - public void setProcessTags(String value) { - ProcessMerlin processMerlin = getProcessObject(); - processMerlin.setTags(value); - setProcessData(processMerlin.toString()); - } - - - public static int runFalconCLI(List<String> args) throws Exception { - args.add(1, "-url"); - args.add(2, prismHelper.getClusterHelper().getHostname()); - LOGGER.info("Going to run falcon jar with args: " + args); - return new FalconCLI().run(args.toArray(new String[]{})); - } -} http://git-wip-us.apache.org/repos/asf/falcon/blob/8e49379d/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/FreqType.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/FreqType.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/FreqType.java deleted file mode 100644 index 46c30ba..0000000 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/FreqType.java +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.falcon.regression.core.enumsAndConstants; - -import org.apache.commons.lang.StringUtils; -import org.apache.falcon.entity.v0.Frequency; -import org.joda.time.DateTime; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; - -/** - * Enum to represent different feed periodicity. - */ -public enum FreqType { - MINUTELY("minutely", "yyyy/MM/dd/HH/mm", "${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}", - "year=${YEAR};month=${MONTH};day=${DAY};hour=${HOUR};minute=${MINUTE}") { - public DateTime addTime(DateTime dateTime, int amount) { - return dateTime.plusMinutes(amount); - } - - @Override - public Frequency.TimeUnit getFalconTimeUnit() { - return Frequency.TimeUnit.minutes; - } - }, - HOURLY("hourly", "yyyy/MM/dd/HH", "${YEAR}/${MONTH}/${DAY}/${HOUR}", - "year=${YEAR};month=${MONTH};day=${DAY};hour=${HOUR}") { - @Override - public DateTime addTime(DateTime dateTime, int amount) { - return dateTime.plusHours(amount); - } - - @Override - public Frequency.TimeUnit getFalconTimeUnit() { - return Frequency.TimeUnit.hours; - } - }, - DAILY("daily", "yyyy/MM/dd", "${YEAR}/${MONTH}/${DAY}", - "year=${YEAR};month=${MONTH};day=${DAY}") { - @Override - public DateTime addTime(DateTime dateTime, int amount) { - return dateTime.plusDays(amount); - } - - @Override - public Frequency.TimeUnit getFalconTimeUnit() { - return Frequency.TimeUnit.days; - } - }, - MONTHLY("monthly", "yyyy/MM", "${YEAR}/${MONTH}", - "year=${YEAR};month=${MONTH}") { - @Override - public DateTime addTime(DateTime dateTime, int amount) { - return dateTime.plusMonths(amount); - } - - @Override - public Frequency.TimeUnit getFalconTimeUnit() { - return Frequency.TimeUnit.months; - } - }, - YEARLY("yearly", "yyyy", "${YEAR}", - "year=${YEAR}") { - @Override - public DateTime addTime(DateTime dateTime, int amount) { - return dateTime.plusYears(amount); - } - - @Override - public Frequency.TimeUnit getFalconTimeUnit() { - throw new UnsupportedOperationException(); - } - }; - - private final String value; - private final String pathValue; - private final String hcatPathValue; - private final DateTimeFormatter formatter; - - private FreqType(String value, String format, String pathValue, String hcatPathValue) { - this.value = value; - formatter = DateTimeFormat.forPattern(format); - this.pathValue = pathValue; - this.hcatPathValue = hcatPathValue; - } - - public String getValue() { - return value; - } - - public String getPathValue() { - return pathValue; - } - - public String getHcatPathValue() { - return hcatPathValue; - } - - public DateTimeFormatter getFormatter() { - return formatter; - } - - public int getDirDepth() { - return StringUtils.countMatches(pathValue, "/"); - } - - public abstract DateTime addTime(DateTime dateTime, int amount); - - public abstract Frequency.TimeUnit getFalconTimeUnit(); -} http://git-wip-us.apache.org/repos/asf/falcon/blob/8e49379d/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java deleted file mode 100644 index 076f68f..0000000 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/MerlinConstants.java +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.falcon.regression.core.enumsAndConstants; - -import org.apache.falcon.regression.core.util.Config; -import org.apache.hadoop.conf.Configuration; -import org.testng.Assert; -import org.apache.log4j.Logger; - -import java.util.HashMap; - -/** - * Class for test constants. - */ -public final class MerlinConstants { - private MerlinConstants() { - } - - private static final Logger LOGGER = Logger.getLogger(MerlinConstants.class); - - public static final String PRISM_URL = Config.getProperty("prism.hostname"); - - public static final String HELP_URL = - Config.getProperty("falcon.help.url", "http://falcon.apache.org/";); - - public static final boolean IS_SECURE = - "kerberos".equals(new Configuration().get("hadoop.security.authentication", "simple")); - - /** Staging location to use in cluster xml. */ - public static final String STAGING_LOCATION = Config.getProperty("merlin.staging.location", - "/tmp/falcon-regression-staging"); - /** Working location to use in cluster xml. */ - public static final String WORKING_LOCATION = Config.getProperty("merlin.working.location", - "/tmp/falcon-regression-working"); - public static final String TEMP_LOCATION = Config.getProperty("merlin.temp.location", "/tmp"); - - public static final String OOZIE_EXAMPLE_LIB = Config.getProperty("merlin.oozie_example_lib", - "https://repo1.maven.org/maven2/org/apache/oozie/oozie-examples/4.1.0/oozie-examples-4.1.0.jar";); - - /** the user that is going to run tests. */ - public static final String CURRENT_USER_NAME = Config.getProperty("current_user_name", - System.getProperty("user.name")); - /** keytab of current user. */ - private static final String CURRENT_USER_KEYTAB_STR = "current_user_keytab"; - /** group of the current user. */ - public static final String CURRENT_USER_GROUP = - Config.getProperty("current_user.group.name", "users"); - - /** a user that does not belong to the group of current user. */ - public static final String DIFFERENT_USER_NAME = Config.getProperty("other.user.name", "root"); - - /** a user that does not belong to the group of current user. */ - public static final String DIFFERENT_USER_GROUP = Config.getProperty("other.user.group", "root"); - - /** falcon super user. */ - public static final String FALCON_SUPER_USER_NAME = - Config.getProperty("falcon.super.user.name", "falcon"); - - /** a user that belongs to falcon super user group but is not FALCON_SUPER_USER_NAME. */ - public static final String FALCON_SUPER_USER2_NAME = - Config.getProperty("falcon.super.user2.name", "falcon2"); - /** a user that has same group as that of current user. */ - private static final String USER_2_NAME_STR = "user2_name"; - private static final String USER_2_KEYTAB_STR = "user2_keytab"; - public static final String USER2_NAME; - private static HashMap<String, String> keyTabMap; - private static HashMap<String, String> passwordMap; - public static final String USER_REALM = Config.getProperty("USER.REALM", ""); - public static final String WASB_CONTAINER = Config.getProperty("wasb.container", ""); - public static final String WASB_SECRET = Config.getProperty("wasb.secret", ""); - public static final String WASB_ACCOUNT = Config.getProperty("wasb.account", ""); - - public static final boolean CLEAN_TESTS_DIR = - Boolean.valueOf(Config.getProperty("clean_tests_dir", "true")); - - public static final boolean IS_DEPRECATE= - Boolean.valueOf(Config.getProperty("is_deprecate", "false")); - - /* initialize keyTabMap */ - static { - final String currentUserKeytab = Config.getProperty(CURRENT_USER_KEYTAB_STR); - final String user2Name = Config.getProperty(USER_2_NAME_STR); - final String user2Keytab = Config.getProperty(USER_2_KEYTAB_STR); - LOGGER.info("CURRENT_USER_NAME: " + CURRENT_USER_NAME); - LOGGER.info("currentUserKeytab: " + currentUserKeytab); - LOGGER.info("user2Name: " + user2Name); - LOGGER.info("user2Keytab: " + user2Keytab); - USER2_NAME = user2Name; - keyTabMap = new HashMap<>(); - keyTabMap.put(CURRENT_USER_NAME, currentUserKeytab); - keyTabMap.put(user2Name, user2Keytab); - keyTabMap.put(FALCON_SUPER_USER_NAME, Config.getProperty("falcon.super.user.keytab")); - keyTabMap.put(FALCON_SUPER_USER2_NAME, Config.getProperty("falcon.super.user2.keytab")); - keyTabMap.put(DIFFERENT_USER_NAME, Config.getProperty("other.user.keytab")); - passwordMap = new HashMap<>(); - passwordMap.put(DIFFERENT_USER_NAME, Config.getProperty("other.user.password")); - } - - public static String getKeytabForUser(String user) { - Assert.assertTrue(keyTabMap.containsKey(user), "Unknown user: " + user); - return keyTabMap.get(user); - } - - public static String getPasswordForUser(String user) { - Assert.assertTrue(passwordMap.containsKey(user), "Unknown user: " + user); - return passwordMap.get(user); - } -} http://git-wip-us.apache.org/repos/asf/falcon/blob/8e49379d/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/ResponseErrors.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/ResponseErrors.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/ResponseErrors.java deleted file mode 100644 index b16edf6..0000000 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/ResponseErrors.java +++ /dev/null @@ -1,43 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.falcon.regression.core.enumsAndConstants; - -/** Class containing response errors of rest requests. */ -public enum ResponseErrors { - - PROCESS_NOT_FOUND("(PROCESS) not found"), - UNPARSEABLE_DATE("Start and End dates cannot be empty for Instance POST apis"), - START_BEFORE_SCHEDULED("is before the entity was scheduled"), - PROCESS_INVALID_RANGE("is not in validity range of process"), - PROCESS_INSTANCE_FAULT("is not a valid instance time on cluster"), - FEED_INVALID_RANGE("is not in validity range for Feed"), - FEED_INSTANCE_FAULT("is not a valid instance for the feed"), - INVALID_INSTANCE_TIME("not a valid instance"); - - private String error; - - ResponseErrors(String error) { - this.error = error; - } - - public String getError() { - return error; - } - -} http://git-wip-us.apache.org/repos/asf/falcon/blob/8e49379d/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/RetentionUnit.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/RetentionUnit.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/RetentionUnit.java deleted file mode 100644 index 92ea25e..0000000 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/enumsAndConstants/RetentionUnit.java +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.falcon.regression.core.enumsAndConstants; - -import org.joda.time.DateTime; - -/** Enum to represent different Retention Units. */ -public enum RetentionUnit { - MINUTES("minutes") { - @Override - public DateTime minusTime(DateTime dateTime, int amount) { - return dateTime.minusMinutes(amount); - } - }, HOURS("hours") { - @Override - public DateTime minusTime(DateTime dateTime, int amount) { - return dateTime.minusHours(amount); - } - }, DAYS("days") { - @Override - public DateTime minusTime(DateTime dateTime, int amount) { - return dateTime.minusDays(amount); - } - }, MONTHS("months") { - @Override - public DateTime minusTime(DateTime dateTime, int amount) { - return dateTime.minusMonths(amount); - } - }, YEARS("years") { - @Override - public DateTime minusTime(DateTime dateTime, int amount) { - return dateTime.minusYears(amount); - } - }; - - private String value; - - private RetentionUnit(String value) { - this.value = value; - } - - public String getValue() { - return value; - } - - public abstract DateTime minusTime(DateTime dateTime, int amount); - -} http://git-wip-us.apache.org/repos/asf/falcon/blob/8e49379d/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/ColoHelper.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/ColoHelper.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/ColoHelper.java deleted file mode 100644 index 4528cb9..0000000 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/ColoHelper.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.falcon.regression.core.helpers; - -import org.apache.falcon.entity.v0.EntityType; -import org.apache.falcon.regression.core.helpers.entity.AbstractEntityHelper; -import org.apache.falcon.regression.core.helpers.entity.EntityHelperFactory; - -/** Helper class to work with a colo. */ -public class ColoHelper { - private final AbstractEntityHelper clusterHelper; - private final AbstractEntityHelper processHelper; - private final AbstractEntityHelper feedHelper; - private String prefix; - - public ColoHelper(String prefix) { - this.prefix = prefix; - clusterHelper = EntityHelperFactory.getEntityHelper(EntityType.CLUSTER, prefix); - processHelper = EntityHelperFactory.getEntityHelper(EntityType.PROCESS, prefix); - feedHelper = EntityHelperFactory.getEntityHelper(EntityType.FEED, prefix); - } - - public AbstractEntityHelper getClusterHelper() { - return clusterHelper; - } - - public AbstractEntityHelper getFeedHelper() { - return feedHelper; - } - - public AbstractEntityHelper getProcessHelper() { - return processHelper; - } - - public String getPrefix() { - return prefix; - } -} http://git-wip-us.apache.org/repos/asf/falcon/blob/8e49379d/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java ---------------------------------------------------------------------- diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java deleted file mode 100644 index fcbb7f9..0000000 --- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/helpers/FalconClientBuilder.java +++ /dev/null @@ -1,190 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.falcon.regression.core.helpers; - -import org.apache.commons.exec.CommandLine; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang3.builder.Builder; -import org.apache.falcon.regression.core.enumsAndConstants.MerlinConstants; -import org.apache.falcon.regression.core.util.Config; -import org.apache.falcon.regression.core.util.OSUtil; - -import java.util.ArrayList; -import java.util.List; - -/** - * FalconClientBuilder is to be used for launching falcon client command. - */ -public final class FalconClientBuilder implements Builder<CommandLine> { - private final String user; - private final CommandLine commandLine; - private final List<String> args; - private final SuType suType; - - private enum SuType { - /** - * Takes care of switching user on linux. Current implemented through sudo. - */ - LIN_SUDO { - @Override - public CommandLine getCommandLine(String forUser) { - return CommandLine.parse("sudo").addArgument("-u") - .addArgument(forUser).addArgument(FALCON_CLIENT_BINARY); - } - @Override - public void addArgsToCommandLine(CommandLine cmdLine, List<String> arguments) { - for (String arg : arguments) { - cmdLine.addArgument(arg); - } - } - }, - /** - * Takes care of switching user on windows. Needs to be implemented. - */ - WIN_SU { - @Override - public CommandLine getCommandLine(String forUser) { - return CommandLine.parse(OSUtil.WIN_SU_BINARY) - .addArgument("-u").addArgument(forUser) - .addArgument("-p").addArgument(MerlinConstants.getPasswordForUser(forUser)) - .addArgument(FALCON_CLIENT_BINARY); - } - @Override - public void addArgsToCommandLine(CommandLine cmdLine, List<String> arguments) { - String lastArg = StringUtils.join(arguments, " "); - cmdLine.addArgument(lastArg, true); - } - }, - /** - * Takes care of the case where no user switch is required. - */ - NONE { - @Override - public CommandLine getCommandLine(String forUser) { - return CommandLine.parse(FALCON_CLIENT_BINARY); - } - @Override - public void addArgsToCommandLine(CommandLine cmdLine, List<String> arguments) { - for (String arg : arguments) { - cmdLine.addArgument(arg); - } - } - }; - - private static final String FALCON_CLIENT_BINARY = - Config.getProperty("falcon.client.binary", "falcon"); - public abstract void addArgsToCommandLine(CommandLine cmdLine, List<String> arguments); - public abstract CommandLine getCommandLine(String forUser); - } - - private FalconClientBuilder(String user) { - this.user = user; - args = new ArrayList<>(); - if (user == null) { - suType = SuType.NONE; - commandLine = suType.getCommandLine(null); - } else { - if (OSUtil.IS_WINDOWS) { - suType = SuType.WIN_SU; - commandLine = suType.getCommandLine(user); - } else { - suType = SuType.LIN_SUDO; - //attempting sudo su - root -c "falcon admin -version" - commandLine = suType.getCommandLine(user); - } - } - } - - /** - * Get an instance of FalconClientBuilder. - * @return instance of FalconClientBuilder - */ - public static FalconClientBuilder getBuilder() { - return new FalconClientBuilder(null); - } - - /** - * Get an instance of FalconClientBuilder for the given user. It would do commandline - * construction in a way that the final command is run as given user. - * @return instance of FalconClientBuilder - */ - public static FalconClientBuilder getBuilder(String user) { - return new FalconClientBuilder(user); - } - - /** - * Add the given argument. - * @param arg argument to be added to builder - * @return this - */ - private FalconClientBuilder addArg(String arg) { - args.add(arg); - return this; - } - - /** - * Create get metrics command. - * @param entityType type of the entity - * @param entityName name of the entity to be deleted - * @return this - */ - public FalconClientBuilder getMetricsCommand(String entityType, String entityName) { - addArg("metadata").addArg("-list"); - addArg("-type").addArg("replication_metrics"); - addArg("-" + entityType).addArg(entityName); - return this; - } - - /** - * Create submit command. - * @param entityType type of the entity - * @param fileName file containing the entity to be submitted - * @return this - */ - public FalconClientBuilder getSubmitCommand(String entityType, String fileName) { - addArg("entity").addArg("-submit"); - addArg("-type").addArg(entityType); - addArg("-file").addArg(fileName); - return this; - } - - /** - * Create delete command. - * @param entityType type of the entity - * @param entityName name of the entity to be deleted - * @return this - */ - public FalconClientBuilder getDeleteCommand(String entityType, String entityName) { - addArg("entity").addArg("-delete"); - addArg("-type").addArg(entityType); - addArg("-name").addArg(entityName); - return this; - } - - - /** - * Build the CommandLine object for this FalconClientBuilder. - * @return instance of CommandLine object - */ - @Override - public CommandLine build() { - suType.addArgsToCommandLine(commandLine, args); - return new CommandLine(commandLine); - } -}
