Repository: falcon Updated Branches: refs/heads/master 98a904c21 -> 7b78c39eb
FALCON-1905 Extension Job Management: IT tests for REST APIs and CLIs Author: yzheng-hortonworks <[email protected]> Reviewers: " Sowmya<[email protected]>,Balu Vellanki <[email protected]>" Closes #129 from yzheng-hortonworks/FALCON-1905 Project: http://git-wip-us.apache.org/repos/asf/falcon/repo Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/7b78c39e Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/7b78c39e Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/7b78c39e Branch: refs/heads/master Commit: 7b78c39ebbddc6ff370e2c758c851ac22225a991 Parents: 98a904c Author: yzheng-hortonworks <[email protected]> Authored: Thu May 12 11:34:37 2016 -0700 Committer: bvellanki <[email protected]> Committed: Thu May 12 11:34:37 2016 -0700 ---------------------------------------------------------------------- .../falcon/resource/ExtensionInstanceList.java | 9 +- .../falcon/resource/ExtensionJobList.java | 9 +- .../falcon/extensions/store/ExtensionStore.java | 2 +- .../apache/falcon/extensions/ExtensionTest.java | 1 + .../store/AbstractTestExtensionStore.java | 23 ++- .../extensions/store/ExtensionStoreTest.java | 1 + webapp/pom.xml | 8 + .../falcon/resource/ExtensionManagerIT.java | 171 +++++++++++++++++++ .../org/apache/falcon/resource/TestContext.java | 53 +++++- .../hdfs-mirroring-property-template.txt | 9 + .../test/resources/hdfs-mirroring-template.xml | 45 +++++ .../hdfs-snapshot-mirroring-template.xml | 45 +++++ .../test/resources/hive-mirroring-template.xml | 45 +++++ webapp/src/test/resources/startup.properties | 1 + 14 files changed, 398 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/client/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java b/client/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java index 65ca4d4..a7ca4e4 100644 --- a/client/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java +++ b/client/src/main/java/org/apache/falcon/resource/ExtensionInstanceList.java @@ -28,14 +28,15 @@ import java.util.List; /** * Instance list of an extension job used for marshalling / unmarshalling with REST calls. */ +//SUSPEND CHECKSTYLE CHECK VisibilityModifierCheck @XmlRootElement @edu.umd.cs.findbugs.annotations.SuppressWarnings({"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) public class ExtensionInstanceList { @XmlElement - private int numEntities; + public int numEntities; @XmlElementWrapper(name = "entitiesSummary") - private List<EntitySummary> entitySummary; + public List<EntitySummary> entitySummary; public ExtensionInstanceList() { numEntities = 0; @@ -70,10 +71,10 @@ public class ExtensionInstanceList { */ public static class EntitySummary { @XmlElement - private EntityList.EntityElement entityProfile; + public EntityList.EntityElement entityProfile; @XmlElement - private InstancesResult.Instance[] instances; + public InstancesResult.Instance[] instances; public EntitySummary() { entityProfile = null; http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/client/src/main/java/org/apache/falcon/resource/ExtensionJobList.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/falcon/resource/ExtensionJobList.java b/client/src/main/java/org/apache/falcon/resource/ExtensionJobList.java index 71f92b8..ca031b5 100644 --- a/client/src/main/java/org/apache/falcon/resource/ExtensionJobList.java +++ b/client/src/main/java/org/apache/falcon/resource/ExtensionJobList.java @@ -27,15 +27,16 @@ import java.util.List; /** * Extension job list used for marshalling / unmarshalling with REST calls. */ +//SUSPEND CHECKSTYLE CHECK VisibilityModifierCheck @XmlRootElement @edu.umd.cs.findbugs.annotations.SuppressWarnings({"EI_EXPOSE_REP", "EI_EXPOSE_REP2"}) public class ExtensionJobList { @XmlElement - private int numJobs; + public int numJobs; @XmlElementWrapper(name = "jobs") - private List<JobElement> job; + public List<JobElement> job; public ExtensionJobList() { numJobs = 0; @@ -71,10 +72,10 @@ public class ExtensionJobList { */ public static class JobElement { @XmlElement - private String jobName; + public String jobName; @XmlElement - private EntityList jobEntities; + public EntityList jobEntities; public JobElement() { jobName = null; http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java ---------------------------------------------------------------------- diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java index 2c2e3c6..4ca432a 100644 --- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java +++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java @@ -56,7 +56,7 @@ public final class ExtensionStore { private static final String RESOURCES_DIR = "resources"; private static final String LIBS_DIR = "libs"; - private static final String EXTENSION_STORE_URI = "extension.store.uri"; + public static final String EXTENSION_STORE_URI = "extension.store.uri"; private static final ExtensionStore STORE = new ExtensionStore(); http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/extensions/src/test/java/org/apache/falcon/extensions/ExtensionTest.java ---------------------------------------------------------------------- diff --git a/extensions/src/test/java/org/apache/falcon/extensions/ExtensionTest.java b/extensions/src/test/java/org/apache/falcon/extensions/ExtensionTest.java index b14d500..21e78d8 100644 --- a/extensions/src/test/java/org/apache/falcon/extensions/ExtensionTest.java +++ b/extensions/src/test/java/org/apache/falcon/extensions/ExtensionTest.java @@ -148,6 +148,7 @@ public class ExtensionTest extends AbstractTestExtensionStore { @BeforeClass public void init() throws Exception { + initExtensionStore(); extension = new Extension(); baseDir = Files.createTempDirectory("test_extensions_hdfs").toFile().getAbsoluteFile(); miniDFSCluster = MiniHdfsClusterUtil.initMiniDfs(MiniHdfsClusterUtil.EXTENSION_TEST_PORT, baseDir); http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java ---------------------------------------------------------------------- diff --git a/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java b/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java index b62b475..fba803f 100644 --- a/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java +++ b/extensions/src/test/java/org/apache/falcon/extensions/store/AbstractTestExtensionStore.java @@ -27,7 +27,6 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.testng.annotations.AfterClass; -import org.testng.annotations.BeforeClass; import java.io.File; import java.io.IOException; @@ -43,19 +42,19 @@ public class AbstractTestExtensionStore { protected ExtensionStore store; private FileSystem fileSystem; - @BeforeClass - public void initConfigStore() throws Exception { - String configPath = new URI(StartupProperties.get().getProperty("extension.store.uri")).getPath(); - extensionStorePath = configPath + "-" + getClass().getName(); - StartupProperties.get().setProperty("extension.store.uri", extensionStorePath); + public void initExtensionStore() throws Exception { + initExtensionStore(this.getClass()); + } + + public void initExtensionStore(Class resourceClass) throws Exception { new ExtensionService().init(); store = ExtensionService.getExtensionStore(); fileSystem = HadoopClientFactory.get().createFalconFileSystem(new Configuration(true)); - - extensionStoreSetup(); + extensionStorePath = new URI(StartupProperties.get().getProperty(ExtensionStore.EXTENSION_STORE_URI)).getPath(); + extensionStoreSetup(resourceClass); } - private void extensionStoreSetup() throws IOException { + private void extensionStoreSetup(Class resourceClass) throws IOException { List<AbstractExtension> extensions = AbstractExtension.getExtensions(); for (AbstractExtension extension : extensions) { String extensionName = extension.getName().toLowerCase(); @@ -86,13 +85,13 @@ public class AbstractTestExtensionStore { Path dstFile = new Path(runTimeResourcePath, extensionName + "-template.xml"); fileSystem.create(dstFile); String srcFile = extensionName + "-template.xml"; - fileSystem.copyFromLocalFile(new Path(getAbsolutePath(srcFile)), dstFile); + fileSystem.copyFromLocalFile(new Path(getAbsolutePath(resourceClass, srcFile)), dstFile); } } - private String getAbsolutePath(String fileName) { - return this.getClass().getResource("/" + fileName).getPath(); + private String getAbsolutePath(Class resourceClass, String fileName) { + return resourceClass.getResource("/" + fileName).getPath(); } http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java ---------------------------------------------------------------------- diff --git a/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java b/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java index 9dbacde..bc04d88 100644 --- a/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java +++ b/extensions/src/test/java/org/apache/falcon/extensions/store/ExtensionStoreTest.java @@ -35,6 +35,7 @@ public class ExtensionStoreTest extends AbstractTestExtensionStore { @BeforeClass public void init() throws Exception { + initExtensionStore(); resourcesMap = ImmutableMap.of( "hdfs-mirroring-template.xml", extensionStorePath + "/hdfs-mirroring/resources/runtime/hdfs-mirroring-template.xml", http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/webapp/pom.xml ---------------------------------------------------------------------- diff --git a/webapp/pom.xml b/webapp/pom.xml index 564a04c..ca1ff18 100644 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -172,6 +172,14 @@ <dependency> <groupId>org.apache.falcon</groupId> + <artifactId>falcon-extensions</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.apache.falcon</groupId> <artifactId>falcon-retention</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java b/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java new file mode 100644 index 0000000..b352be4 --- /dev/null +++ b/webapp/src/test/java/org/apache/falcon/resource/ExtensionManagerIT.java @@ -0,0 +1,171 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.falcon.resource; + +import org.apache.falcon.extensions.ExtensionProperties; +import org.apache.falcon.extensions.mirroring.hdfs.HdfsMirroringExtensionProperties; +import org.apache.falcon.extensions.store.AbstractTestExtensionStore; +import org.testng.Assert; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.util.HashMap; +import java.util.Map; + +/** + * Unit tests for org.apache.falcon.extensions.ExtensionManager. + */ +public class ExtensionManagerIT extends AbstractTestExtensionStore { + private static final String HDFS_MIRRORING_PROPERTY_TEMPLATE = "/hdfs-mirroring-property-template.txt"; + private static final String JOB_NAME_1 = "hdfs-mirroring-job-1"; + private static final String JOB_NAME_2 = "hdfs-mirroring-job-2"; + private static final String CLUSTER_NAME = "primaryCluster"; + private static final String START_TIME_1 = "2016-05-03T00:00Z"; + private static final String START_TIME_2 = "2016-05-01T00:00Z"; + private static final String FREQUENCY = "days(1)"; + private static final String SOURCE_DIR = "/apps/falcon/demo/input-{year}-{month}-{day}"; + private static final String TARGET_DIR = "/apps/falcon/demo/output-{year}-{month}-{day}"; + + private final TestContext context = new TestContext(); + + @BeforeClass + public void prepare() throws Exception { + TestContext.prepare(); + initExtensionStore(TestContext.class); + } + + @AfterClass + public void tearDown() throws Exception { + TestContext.deleteEntitiesFromStore(); + } + + @Test + public void testTrustedExtensionJob() throws Exception { + Map<String, String> overlay = context.getUniqueOverlay(); + String endTime = context.getProcessEndTime(); + + // submit cluster + overlay.put("cluster", CLUSTER_NAME); + String filePath = TestContext.overlayParametersOverTemplate(TestContext.CLUSTER_TEMPLATE, overlay); + System.out.println("entity -submit -type cluster -file " + filePath); + Assert.assertEquals(TestContext.executeWithURL("entity -submit -type cluster -file " + filePath), 0); + + // submit + System.out.println(getHDFSMirroringProperty(JOB_NAME_1, START_TIME_1, endTime).toString()); + filePath = TestContext.overlayParametersOverTemplate( + TestContext.getTempFile("property", "txt"), + HDFS_MIRRORING_PROPERTY_TEMPLATE, + getHDFSMirroringProperty(JOB_NAME_1, START_TIME_1, endTime)); + System.out.println("extension -submit -extensionName hdfs-mirroring -file " + filePath); + Assert.assertEquals(TestContext.executeWithURL( + "extension -submit -extensionName hdfs-mirroring -file " + filePath), 0); + + // schedule + System.out.println("extension -schedule -jobName " + JOB_NAME_1); + Assert.assertEquals(TestContext.executeWithURL("extension -schedule -jobName " + JOB_NAME_1), 0); + + // submit and schedule + filePath = TestContext.overlayParametersOverTemplate( + TestContext.getTempFile("property", "txt"), + HDFS_MIRRORING_PROPERTY_TEMPLATE, + getHDFSMirroringProperty(JOB_NAME_2, START_TIME_1, endTime)); + System.out.println("extension -submitAndSchedule -extensionName hdfs-mirroring -file " + filePath); + Assert.assertEquals(TestContext.executeWithURL( + "extension -submitAndSchedule -extensionName hdfs-mirroring -file " + filePath), 0); + + // list extension jobs + System.out.println("extension -list -extensionName hdfs-mirroring -fields status,clusters,tags"); + Assert.assertEquals(TestContext.executeWithURL( + "extension -list -extensionName hdfs-mirroring -fields status,clusters,tags"), 0); + + // validate job list results + ExtensionJobList jobs = context.getExtensionJobs("hdfs-mirroring", null, null, null, null, null); + Assert.assertEquals(jobs.numJobs, 2); + Assert.assertEquals(jobs.job.get(0).jobName, JOB_NAME_1); + Assert.assertEquals(jobs.job.get(1).jobName, JOB_NAME_2); + + // list extension job instances + System.out.println("extension -instances -jobName " + JOB_NAME_1 + " -fields status,clusters,tags"); + Assert.assertEquals(TestContext.executeWithURL( + "extension -instances -jobName " + JOB_NAME_1 + " -fields status,clusters,tags"), 0); + System.out.println("extension -instances -jobName " + JOB_NAME_2 + " -fields status,clusters,tags"); + Assert.assertEquals(TestContext.executeWithURL( + "extension -instances -jobName " + JOB_NAME_2 + " -fields status,clusters,tags"), 0); + + // validate instance list results + ExtensionInstanceList instanceList = context.getExtensionInstances(JOB_NAME_1, START_TIME_1, endTime, "RUNNING", + null, null, null, null, null, null); + System.out.println("Validate running instances of extension job " + JOB_NAME_1 + ": \n" + + instanceList.toString()); + Assert.assertEquals(instanceList.numEntities, 1); + Assert.assertEquals(instanceList.entitySummary.get(0).instances.length, 1); + instanceList = context.getExtensionInstances(JOB_NAME_2, START_TIME_1, endTime, "RUNNING", + null, null, null, null, null, null); + System.out.println("Validate running instances of extension job " + JOB_NAME_2 + ": \n" + + instanceList.toString()); + Assert.assertEquals(instanceList.numEntities, 1); + Assert.assertEquals(instanceList.entitySummary.get(0).instances.length, 1); + + // suspend + System.out.println("extension -suspend -jobName " + JOB_NAME_1); + Assert.assertEquals(TestContext.executeWithURL("extension -suspend -jobName " + JOB_NAME_1), 0); + + // resume + System.out.println("extension -resume -jobName " + JOB_NAME_1); + Assert.assertEquals(TestContext.executeWithURL("extension -resume -jobName " + JOB_NAME_1), 0); + + // delete + System.out.println("extension -delete -jobName " + JOB_NAME_1); + Assert.assertEquals(TestContext.executeWithURL("extension -delete -jobName " + JOB_NAME_1), 0); + + // update + filePath = TestContext.overlayParametersOverTemplate( + TestContext.getTempFile("property", "txt"), + HDFS_MIRRORING_PROPERTY_TEMPLATE, + getHDFSMirroringProperty(JOB_NAME_2, START_TIME_2, endTime)); + System.out.println("extension -update -extensionName hdfs-mirroring -file " + filePath); + Assert.assertEquals(TestContext.executeWithURL( + "extension -update -extensionName hdfs-mirroring -file " + filePath), 0); + + // validate + System.out.println("extension -validate -extensionName hdfs-mirroring -file " + filePath); + Assert.assertEquals(TestContext.executeWithURL( + "extension -validate -extensionName hdfs-mirroring -file " + filePath), 0); + + // failure case: no file input + System.out.println("extension -submitAndSchedule -extensionName hdfs-mirroring"); + Assert.assertEquals(TestContext.executeWithURL( + "extension -submitAndSchedule -extensionName hdfs-mirroring"), -1); + } + + private Map<String, String> getHDFSMirroringProperty(String jobName, String start, String end) { + Map<String, String> properties = new HashMap<>(); + properties.put(ExtensionProperties.JOB_NAME.getName(), jobName); + properties.put(ExtensionProperties.CLUSTER_NAME.getName(), CLUSTER_NAME); + properties.put(ExtensionProperties.VALIDITY_START.getName(), start); + properties.put(ExtensionProperties.VALIDITY_END.getName(), end); + properties.put(ExtensionProperties.FREQUENCY.getName(), FREQUENCY); + properties.put(HdfsMirroringExtensionProperties.SOURCE_DIR.getName(), SOURCE_DIR); + properties.put(HdfsMirroringExtensionProperties.SOURCE_CLUSTER.getName(), CLUSTER_NAME); + properties.put(HdfsMirroringExtensionProperties.TARGET_DIR.getName(), TARGET_DIR); + properties.put(HdfsMirroringExtensionProperties.TARGET_CLUSTER.getName(), CLUSTER_NAME); + return properties; + } +} http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/webapp/src/test/java/org/apache/falcon/resource/TestContext.java ---------------------------------------------------------------------- diff --git a/webapp/src/test/java/org/apache/falcon/resource/TestContext.java b/webapp/src/test/java/org/apache/falcon/resource/TestContext.java index 3cf5c18..5412608 100644 --- a/webapp/src/test/java/org/apache/falcon/resource/TestContext.java +++ b/webapp/src/test/java/org/apache/falcon/resource/TestContext.java @@ -61,6 +61,7 @@ import javax.net.ssl.SSLContext; import javax.net.ssl.SSLSession; import javax.net.ssl.TrustManager; import javax.servlet.ServletInputStream; +import javax.ws.rs.HttpMethod; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriBuilder; @@ -288,6 +289,41 @@ public class TestContext extends AbstractTestContext { }; } + public ExtensionJobList getExtensionJobs(String extensionName, String fields, String sortOrder, String offset, + String resultsPerPage, String doAsUser) { + WebResource resource = this.service.path("api/extension/list/" + extensionName); + resource = addQueryParam(resource, "doAs", doAsUser); + resource = addQueryParam(resource, "fields", fields); + resource = addQueryParam(resource, "sortOrder", sortOrder); + resource = addQueryParam(resource, "offset", offset); + resource = addQueryParam(resource, "numResults", resultsPerPage); + ClientResponse response = resource.header("Cookie", AUTH_COOKIE_EQ + authenticationToken) + .accept(MediaType.APPLICATION_JSON).type(MediaType.TEXT_XML) + .method(HttpMethod.GET, ClientResponse.class); + return response.getEntity(ExtensionJobList.class); + } + + //SUSPEND CHECKSTYLE CHECK ParameterNumberCheck + public ExtensionInstanceList getExtensionInstances(String jobName, String nominalStart, String nominalEnd, + String instanceStatus, String fields, String orderBy, + String sortOrder, String offset, String resultsPerPage, + String doAsUser) { + WebResource resource = this.service.path("api/extension/instances/" + jobName); + resource = addQueryParam(resource, "start", nominalStart); + resource = addQueryParam(resource, "end", nominalEnd); + resource = addQueryParam(resource, "instanceStatus", instanceStatus); + resource = addQueryParam(resource, "doAs", doAsUser); + resource = addQueryParam(resource, "fields", fields); + resource = addQueryParam(resource, "orderBy", orderBy); + resource = addQueryParam(resource, "sortOrder", sortOrder); + resource = addQueryParam(resource, "offset", offset); + resource = addQueryParam(resource, "numResults", resultsPerPage); + ClientResponse response = resource.header("Cookie", AUTH_COOKIE_EQ + authenticationToken) + .accept(MediaType.APPLICATION_JSON).type(MediaType.TEXT_XML) + .method(HttpMethod.GET, ClientResponse.class); + return response.getEntity(ExtensionInstanceList.class); + } + public ClientResponse submitAndSchedule(String template, Map<String, String> overlay, EntityType entityType) throws Exception { return submitAndSchedule(template, overlay, entityType, null, "", null); @@ -448,8 +484,12 @@ public class TestContext extends AbstractTestContext { public static String overlayParametersOverTemplate(String template, Map<String, String> overlay) throws IOException { - File tmpFile = getTempFile(); - OutputStream out = new FileOutputStream(tmpFile); + return overlayParametersOverTemplate(getTempFile(), template, overlay); + } + + public static String overlayParametersOverTemplate(File file, String template, + Map<String, String> overlay) throws IOException { + OutputStream out = new FileOutputStream(file); InputStreamReader in; InputStream resourceAsStream = TestContext.class.getResourceAsStream(template); @@ -472,7 +512,7 @@ public class TestContext extends AbstractTestContext { } reader.close(); out.close(); - return tmpFile.getAbsolutePath(); + return file.getAbsolutePath(); } @SuppressWarnings("ResultOfMethodCallIgnored") @@ -579,4 +619,11 @@ public class TestContext extends AbstractTestContext { public static int executeWithURL(String command) throws Exception { return new FalconCLI().run((command + " -url " + TestContext.BASE_URL).split("\\s+")); } + + private WebResource addQueryParam(WebResource resource, String key, String value) { + if (StringUtils.isEmpty(value)) { + return resource; + } + return resource.queryParam(key, value); + } } http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/webapp/src/test/resources/hdfs-mirroring-property-template.txt ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/hdfs-mirroring-property-template.txt b/webapp/src/test/resources/hdfs-mirroring-property-template.txt new file mode 100644 index 0000000..c01bc01 --- /dev/null +++ b/webapp/src/test/resources/hdfs-mirroring-property-template.txt @@ -0,0 +1,9 @@ +jobName=##jobName## +jobClusterName=##jobClusterName## +jobValidityStart=##jobValidityStart## +jobValidityEnd=##jobValidityEnd## +jobFrequency=##jobFrequency## +sourceDir=##sourceDir## +sourceCluster=##sourceCluster## +targetDir=##targetDir## +targetCluster=##targetCluster## http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/webapp/src/test/resources/hdfs-mirroring-template.xml ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/hdfs-mirroring-template.xml b/webapp/src/test/resources/hdfs-mirroring-template.xml new file mode 100644 index 0000000..6c35c5b --- /dev/null +++ b/webapp/src/test/resources/hdfs-mirroring-template.xml @@ -0,0 +1,45 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<process name="##job.name##" xmlns="uri:falcon:process:0.1"> + <clusters> + <!-- source --> + <cluster name="##job.cluster.name##"> + <validity end="##job.validity.end##" start="##job.validity.start##"/> + </cluster> + </clusters> + + <tags/> + + <parallel>1</parallel> + <!-- Dir replication needs to run only once to catch up --> + <order>LAST_ONLY</order> + <frequency>##job.frequency##</frequency> + <timezone>##job.timezone##</timezone> + + <properties> + <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/> + </properties> + + <workflow name="##job.workflow.name##" engine="##job.workflow.engine##" + path="##job.workflow.path##" lib="##job.workflow.lib.path##"/> + <retry policy="##job.retry.policy##" delay="##job.retry.delay##" attempts="3"/> + <notification type="##job.notification.type##" to="##job.notification.receivers##"/> + <ACL/> +</process> http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/webapp/src/test/resources/hdfs-snapshot-mirroring-template.xml ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/hdfs-snapshot-mirroring-template.xml b/webapp/src/test/resources/hdfs-snapshot-mirroring-template.xml new file mode 100644 index 0000000..29131da --- /dev/null +++ b/webapp/src/test/resources/hdfs-snapshot-mirroring-template.xml @@ -0,0 +1,45 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<process name="##jobName##" xmlns="uri:falcon:process:0.1"> + <clusters> + <!-- source --> + <cluster name="##jobClusterName##"> + <validity end="##jobValidityEnd##" start="##jobValidityStart##"/> + </cluster> + </clusters> + + <tags/> + + <parallel>1</parallel> + <!-- Replication needs to run only once to catch up --> + <order>LAST_ONLY</order> + <frequency>##jobFrequency##</frequency> + <timezone>##jobTimezone##</timezone> + + <properties> + <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/> + </properties> + + <workflow name="##jobWorkflowName##" engine="##jobWorkflowEngine##" + path="##jobWorkflowPath##" lib="##jobWorkflowLibPath##"/> + <retry policy="##jobRetryPolicy##" delay="##jobRetryDelay##" attempts="3"/> + <notification type="##jobNotificationType##" to="##jobNotificationReceivers##"/> + <ACL/> +</process> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/webapp/src/test/resources/hive-mirroring-template.xml ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/hive-mirroring-template.xml b/webapp/src/test/resources/hive-mirroring-template.xml new file mode 100644 index 0000000..9f28991 --- /dev/null +++ b/webapp/src/test/resources/hive-mirroring-template.xml @@ -0,0 +1,45 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<process name="##name##" xmlns="uri:falcon:process:0.1"> + <clusters> + <!-- source --> + <cluster name="##cluster.name##"> + <validity end="##cluster.validity.end##" start="##cluster.validity.start##"/> + </cluster> + </clusters> + + <tags/> + + <parallel>1</parallel> + <!-- Replication needs to run only once to catch up --> + <order>LAST_ONLY</order> + <frequency>##process.frequency##</frequency> + <timezone>UTC</timezone> + + <properties> + <property name="oozie.wf.subworkflow.classpath.inheritance" value="true"/> + </properties> + + <workflow name="##workflow.name##" engine="oozie" + path="/apps/data-mirroring/workflows/hive-disaster-recovery-workflow.xml" lib="##workflow.lib.path##"/> + <retry policy="##retry.policy##" delay="##retry.delay##" attempts="3"/> + <notification type="##notification.type##" to="##notification.receivers##"/> + <ACL/> +</process> http://git-wip-us.apache.org/repos/asf/falcon/blob/7b78c39e/webapp/src/test/resources/startup.properties ---------------------------------------------------------------------- diff --git a/webapp/src/test/resources/startup.properties b/webapp/src/test/resources/startup.properties index 58018f1..c7d7c87 100644 --- a/webapp/src/test/resources/startup.properties +++ b/webapp/src/test/resources/startup.properties @@ -68,6 +68,7 @@ debug.retry.recorder.path=${user.dir}/target/retry debug.libext.feed.retention.paths=${falcon.libext} debug.libext.feed.replication.paths=${falcon.libext} debug.libext.process.paths=${falcon.libext} +debug.extension.store.uri=jail://global:00/extension/store *.falcon.cleanup.service.frequency=minutes(5)
