Tests for Azkaban Orchestrator
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/ee3e5481 Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/ee3e5481 Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/ee3e5481 Branch: refs/heads/master Commit: ee3e5481a1ef538fd5d5610e30254cb1fc80cad1 Parents: 0bb5139 Author: Abhishek Tiwari <[email protected]> Authored: Wed Aug 30 04:43:20 2017 -0700 Committer: Abhishek Tiwari <[email protected]> Committed: Wed Aug 30 04:43:20 2017 -0700 ---------------------------------------------------------------------- gobblin-modules/gobblin-azkaban/build.gradle | 3 + .../orchestration/AzkabanAjaxAPIClient.java | 7 +- .../orchestration/AzkabanProjectConfig.java | 11 +- .../orchestration/AzkabanAjaxAPIClientTest.java | 97 +++++++++++++++++ .../orchestration/AzkabanProjectConfigTest.java | 109 +++++++++++++++++++ .../src/test/resources/reference.conf | 15 +++ 6 files changed, 238 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ee3e5481/gobblin-modules/gobblin-azkaban/build.gradle ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-azkaban/build.gradle b/gobblin-modules/gobblin-azkaban/build.gradle index 4bebecc..2f1dde6 100644 --- a/gobblin-modules/gobblin-azkaban/build.gradle +++ b/gobblin-modules/gobblin-azkaban/build.gradle @@ -41,6 +41,9 @@ dependencies { compile externalDependency.typesafeConfig compile externalDependency.hadoopYarnApi compile externalDependency.findBugsAnnotations + + testCompile externalDependency.mockito + testCompile externalDependency.powermock } test { http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ee3e5481/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java index d0b8471..90bf005 100644 --- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java +++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java @@ -51,6 +51,7 @@ import org.apache.http.impl.client.HttpClients; import org.apache.http.ssl.SSLContextBuilder; import org.apache.http.ssl.TrustStrategy; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Splitter; import com.google.common.collect.Maps; import com.google.gson.JsonElement; @@ -345,14 +346,16 @@ public class AzkabanAjaxAPIClient { return postRequest; } - private static Map<String, String> executeGetRequest(HttpGet getRequest) throws IOException { + @VisibleForTesting + protected static Map<String, String> executeGetRequest(HttpGet getRequest) throws IOException { // Make the call, get response @Cleanup CloseableHttpClient httpClient = getHttpClient(); HttpResponse response = httpClient.execute(getRequest); return handleResponse(response); } - private static Map<String, String> executePostRequest(HttpPost postRequest) throws IOException { + @VisibleForTesting + protected static Map<String, String> executePostRequest(HttpPost postRequest) throws IOException { // Make the call, get response @Cleanup CloseableHttpClient httpClient = getHttpClient(); HttpResponse response = httpClient.execute(postRequest); http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ee3e5481/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java index 2bac65d..b99683d 100644 --- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java +++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java @@ -24,6 +24,7 @@ import lombok.Builder; import lombok.Getter; import lombok.ToString; +import org.apache.commons.lang.StringUtils; import org.apache.gobblin.runtime.api.JobSpec; import org.apache.gobblin.util.ConfigUtils; @@ -76,8 +77,14 @@ public class AzkabanProjectConfig { this.azkabanZipJarNames = Optional.ofNullable(ConfigUtils.getStringList(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_JAR_NAMES_KEY)); this.azkabanZipJarUrlTemplate = Optional.ofNullable(ConfigUtils.getString(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_JAR_URL_TEMPLATE_KEY, null)); this.azkabanZipJarVersion = Optional.ofNullable(ConfigUtils.getString(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_JAR_VERSION_KEY, null)); - this.azkabanZipAdditionalFiles = Optional.ofNullable( - ConfigUtils.getStringList(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_ADDITIONAL_FILE_URLS_KEY)); + if (config.hasPath(ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_ADDITIONAL_FILE_URLS_KEY) && + StringUtils.isNotBlank(config.getString(ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_ADDITIONAL_FILE_URLS_KEY))) { + this.azkabanZipAdditionalFiles = Optional.ofNullable( + ConfigUtils.getStringList(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_ADDITIONAL_FILE_URLS_KEY)); + } else { + this.azkabanZipAdditionalFiles = Optional.empty(); + } + this.failIfJarNotFound = ConfigUtils.getBoolean(config, ServiceAzkabanConfigKeys.AZKABAN_PROJECT_ZIP_FAIL_IF_JARNOTFOUND_KEY, false); } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ee3e5481/gobblin-modules/gobblin-azkaban/src/test/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClientTest.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-azkaban/src/test/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClientTest.java b/gobblin-modules/gobblin-azkaban/src/test/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClientTest.java new file mode 100644 index 0000000..d7dda91 --- /dev/null +++ b/gobblin-modules/gobblin-azkaban/src/test/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClientTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gobblin.service.modules.orchestration; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; + +import lombok.extern.slf4j.Slf4j; + +import org.testng.Assert; +import org.testng.annotations.Test; + + +@Slf4j +@Test(groups = { "org.apache.gobblin.service.modules.orchestration" }) +public class AzkabanAjaxAPIClientTest { + + @Test + public void testCurrentTimeWithinWindow() + throws ParseException { + // Current hour + Calendar now = Calendar.getInstance(); + int currentHour = now.get(Calendar.HOUR_OF_DAY); + + // Generate a window encapsulating the current time + int windowStartInHours = currentHour < 2 ? 24 + (currentHour - 2) : currentHour - 2; + int windowEndInHours = (currentHour + 3) % 24; + int delayInMinutes = 0; + + // Get computed scheduled time + String outputScheduledString = + AzkabanAjaxAPIClient.getScheduledTimeInAzkabanFormat(windowStartInHours, windowEndInHours, delayInMinutes); + + // Verify that output schedule time is within window + Assert.assertTrue(isWithinWindow(windowStartInHours, windowEndInHours, outputScheduledString)); + } + + @Test + public void testCurrentTimeOutsideWindow() + throws ParseException { + // Current hour + Calendar now = Calendar.getInstance(); + int currentHour = now.get(Calendar.HOUR_OF_DAY); + + // Generate a window NOT encapsulating the current time + int windowStartInHours = currentHour > 10 ? 1 : 11; + int windowEndInHours = currentHour > 10 ? 6 : 16; + int delayInMinutes = 0; + + // Get computed scheduled time + String outputScheduledString = + AzkabanAjaxAPIClient.getScheduledTimeInAzkabanFormat(windowStartInHours, windowEndInHours, delayInMinutes); + + // Verify that output schedule time is within window + Assert.assertTrue(isWithinWindow(windowStartInHours, windowEndInHours, outputScheduledString)); + } + + private boolean isWithinWindow(int windowStartInHours, int windowEndInHours, String outputScheduledString) + throws ParseException { + Calendar windowStart = Calendar.getInstance(); + windowStart.set(Calendar.HOUR_OF_DAY, windowStartInHours); + windowStart.set(Calendar.MINUTE, 0); + windowStart.set(Calendar.SECOND, 0); + + Calendar windowEnd = Calendar.getInstance(); + windowEnd.set(Calendar.HOUR_OF_DAY, windowEndInHours); + windowEnd.set(Calendar.MINUTE, 0); + windowEnd.set(Calendar.SECOND, 0); + + Date outputDate = new SimpleDateFormat("hh,mm,a,z").parse(outputScheduledString); + Calendar receivedTime = Calendar.getInstance(); + receivedTime.set(Calendar.HOUR_OF_DAY, Integer.parseInt(new SimpleDateFormat("HH").format(outputDate))); + receivedTime.set(Calendar.MINUTE, Integer.parseInt(new SimpleDateFormat("mm").format(outputDate))); + + log.info("Window start time is: " + new SimpleDateFormat("MM/dd/yyyy hh,mm,a,z").format(windowStart.getTime())); + log.info("Window end time is: " + new SimpleDateFormat("MM/dd/yyyy hh,mm,a,z").format(windowEnd.getTime())); + log.info("Output time is: " + new SimpleDateFormat("MM/dd/yyyy hh,mm,a,z").format(receivedTime.getTime())); + + return receivedTime.after(windowStart) && receivedTime.before(windowEnd); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ee3e5481/gobblin-modules/gobblin-azkaban/src/test/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfigTest.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-azkaban/src/test/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfigTest.java b/gobblin-modules/gobblin-azkaban/src/test/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfigTest.java new file mode 100644 index 0000000..9e189ab --- /dev/null +++ b/gobblin-modules/gobblin-azkaban/src/test/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfigTest.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.gobblin.service.modules.orchestration; + +import java.net.URI; +import java.util.Properties; + +import lombok.extern.slf4j.Slf4j; + +import org.apache.gobblin.runtime.api.JobSpec; +import org.apache.gobblin.util.ConfigUtils; +import org.testng.Assert; +import org.testng.annotations.Test; + +import com.google.common.base.Optional; + + +@Slf4j +@Test(groups = { "org.apache.gobblin.service.modules.orchestration" }) +public class AzkabanProjectConfigTest { + + @Test + public void testProjectNameDefault() throws Exception { + String expectedProjectName = "GobblinService__uri"; + + Properties properties = new Properties(); + JobSpec jobSpec = new JobSpec(new URI("uri"), "0.0", "test job spec", + ConfigUtils.propertiesToConfig(properties), properties, Optional.absent()); + AzkabanProjectConfig azkabanProjectConfig = new AzkabanProjectConfig(jobSpec); + + String actualProjectName = azkabanProjectConfig.getAzkabanProjectName(); + + Assert.assertEquals(actualProjectName, expectedProjectName); + } + + @Test + public void testProjectNameWithConfig() throws Exception { + String expectedProjectName = "randomPrefix_http___localhost_8000_context"; + + Properties properties = new Properties(); + properties.setProperty("gobblin.service.azkaban.project.namePrefix", "randomPrefix"); + JobSpec jobSpec = new JobSpec(new URI("http://localhost:8000/context";), "0.0", "test job spec", + ConfigUtils.propertiesToConfig(properties), properties, Optional.absent()); + AzkabanProjectConfig azkabanProjectConfig = new AzkabanProjectConfig(jobSpec); + + String actualProjectName = azkabanProjectConfig.getAzkabanProjectName(); + + Assert.assertEquals(actualProjectName, expectedProjectName); + } + + @Test + public void testProjectNameWithReallyLongName() throws Exception { + String expectedProjectName = "randomPrefixWithReallyLongName_http___localhost_8000__55490420"; + + Properties properties = new Properties(); + properties.setProperty("gobblin.service.azkaban.project.namePrefix", "randomPrefixWithReallyLongName"); + JobSpec jobSpec = new JobSpec(new URI("http://localhost:8000/context/that-keeps-expanding-and-explanding";), + "0.0", "test job spec", ConfigUtils.propertiesToConfig(properties), properties, Optional.absent()); + AzkabanProjectConfig azkabanProjectConfig = new AzkabanProjectConfig(jobSpec); + + String actualProjectName = azkabanProjectConfig.getAzkabanProjectName(); + + Assert.assertEquals(actualProjectName, expectedProjectName); + } + + @Test + public void testProjectZipFileName() throws Exception { + String expectedZipFileName = "randomPrefix_http___localhost_8000_context.zip"; + + Properties properties = new Properties(); + properties.setProperty("gobblin.service.azkaban.project.namePrefix", "randomPrefix"); + JobSpec jobSpec = new JobSpec(new URI("http://localhost:8000/context";), "0.0", "test job spec", + ConfigUtils.propertiesToConfig(properties), properties, Optional.absent()); + AzkabanProjectConfig azkabanProjectConfig = new AzkabanProjectConfig(jobSpec); + + String actualZipFileName = azkabanProjectConfig.getAzkabanProjectZipFilename(); + + Assert.assertEquals(actualZipFileName, expectedZipFileName); + } + + @Test + public void testProjectZipFileNameForLongName() throws Exception { + String expectedZipFileName = "randomPrefixWithReallyLongName_http___localhost_8000__55490420.zip"; + + Properties properties = new Properties(); + properties.setProperty("gobblin.service.azkaban.project.namePrefix", "randomPrefixWithReallyLongName"); + JobSpec jobSpec = new JobSpec(new URI("http://localhost:8000/context/that-keeps-expanding-and-explanding";), + "0.0", "test job spec", ConfigUtils.propertiesToConfig(properties), properties, Optional.absent()); + AzkabanProjectConfig azkabanProjectConfig = new AzkabanProjectConfig(jobSpec); + + String actualZipFileName = azkabanProjectConfig.getAzkabanProjectZipFilename(); + + Assert.assertEquals(actualZipFileName, expectedZipFileName); + } +} http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/ee3e5481/gobblin-modules/gobblin-azkaban/src/test/resources/reference.conf ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-azkaban/src/test/resources/reference.conf b/gobblin-modules/gobblin-azkaban/src/test/resources/reference.conf new file mode 100644 index 0000000..936b8a1 --- /dev/null +++ b/gobblin-modules/gobblin-azkaban/src/test/resources/reference.conf @@ -0,0 +1,15 @@ +# Sample configuration properties with default values + +# Cluster configuration properties +gobblin.cluster.app.name=GobblinStandaloneCluster +gobblin.cluster.email.notification.on.shutdown=false +gobblin.cluster.helix.instance.max.retries=2 +gobblin.cluster.work.dir=/tmp/gobblin-cluster + +# Helix/Zookeeper configuration properties +gobblin.cluster.helix.cluster.name=GobblinStandaloneCluster +gobblin.cluster.zk.connection.string="localhost:2181" + +fs.uri="file:///" + +job.execinfo.server.enabled=false
