Azkaban orchestrator findbugs fix, addition of default config fallback, updated config values
Project: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/commit/b0c96aca Tree: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/tree/b0c96aca Diff: http://git-wip-us.apache.org/repos/asf/incubator-gobblin/diff/b0c96aca Branch: refs/heads/master Commit: b0c96acabf9056b8caa8e565f737be989f449056 Parents: ee3e548 Author: Abhishek Tiwari <[email protected]> Authored: Wed Aug 30 08:07:38 2017 -0700 Committer: Abhishek Tiwari <[email protected]> Committed: Wed Aug 30 08:07:38 2017 -0700 ---------------------------------------------------------------------- conf/service/log4j-cluster.properties | 27 ------------------ conf/service/log4j-service.properties | 27 ++++++++++++++++++ .../orchestration/AzkabanAjaxAPIClient.java | 5 +++- .../modules/orchestration/AzkabanJobHelper.java | 30 ++++++++++++++++++-- .../orchestration/AzkabanProjectConfig.java | 4 +-- .../AzkabanSpecExecutorInstance.java | 4 ++- .../AzkabanSpecExecutorInstanceProducer.java | 6 ++-- .../orchestration/ServiceAzkabanConfigKeys.java | 1 + .../main/resources/default-service-azkaban.conf | 2 +- .../gobblin/service/ServiceConfigKeys.java | 2 ++ .../modules/core/GobblinServiceManager.java | 1 + 11 files changed, 70 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/conf/service/log4j-cluster.properties ---------------------------------------------------------------------- diff --git a/conf/service/log4j-cluster.properties b/conf/service/log4j-cluster.properties deleted file mode 100755 index a7ffb68..0000000 --- a/conf/service/log4j-cluster.properties +++ /dev/null @@ -1,27 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# log4j configuration used during build and unit tests - -log4j.rootLogger=info,stdout -log4j.threshhold=ALL -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss z} %-5p [%t] %C %X{tableName} - %m%n - -# Suppressed loggers -log4j.logger.org.apache.helix.controller.GenericHelixController=ERROR -log4j.logger.org.apache.helix.controller.stages=ERROR -log4j.logger.org.apache.helix.controller.strategy.AutoRebalanceStrategy=ERROR -log4j.logger.org.apache.helix.manager.zk=ERROR -log4j.logger.org.apache.helix.monitoring.mbeans.ClusterStatusMonitor=ERROR -log4j.logger.org.apache.helix.store.zk.AutoFallbackPropertyStore=ERROR \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/conf/service/log4j-service.properties ---------------------------------------------------------------------- diff --git a/conf/service/log4j-service.properties b/conf/service/log4j-service.properties new file mode 100755 index 0000000..a7ffb68 --- /dev/null +++ b/conf/service/log4j-service.properties @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# log4j configuration used during build and unit tests + +log4j.rootLogger=info,stdout +log4j.threshhold=ALL +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss z} %-5p [%t] %C %X{tableName} - %m%n + +# Suppressed loggers +log4j.logger.org.apache.helix.controller.GenericHelixController=ERROR +log4j.logger.org.apache.helix.controller.stages=ERROR +log4j.logger.org.apache.helix.controller.strategy.AutoRebalanceStrategy=ERROR +log4j.logger.org.apache.helix.manager.zk=ERROR +log4j.logger.org.apache.helix.monitoring.mbeans.ClusterStatusMonitor=ERROR +log4j.logger.org.apache.helix.store.zk.AutoFallbackPropertyStore=ERROR \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java index 90bf005..9c3cee4 100644 --- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java +++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanAjaxAPIClient.java @@ -308,7 +308,7 @@ public class AzkabanAjaxAPIClient { throws IOException { Map<String, String> params = Maps.newHashMap(); params.put("ajax", "executeFlow"); - params.put("projectName", azkabanProjectConfig.getAzkabanProjectName()); + params.put("project", azkabanProjectConfig.getAzkabanProjectName()); params.put("flow", azkabanProjectConfig.getAzkabanProjectFlowName()); executePostRequest(preparePostRequest(azkabanProjectConfig.getAzkabanServerUrl() + "/executor", sessionId, params)); @@ -454,6 +454,9 @@ public class AzkabanAjaxAPIClient { * @param delayMinutes If current time is within window, then additional delay for bootstrapping if desired * @return Scheduled time string of the format hh,mm,a,z */ + @edu.umd.cs.findbugs.annotations.SuppressWarnings( + value = "DMI_RANDOM_USED_ONLY_ONCE", + justification = "As expected for randomization") public static String getScheduledTimeInAzkabanFormat(int windowStartHour, int windowEndHour, int delayMinutes) { // Validate if (windowStartHour < 0 || windowEndHour > 23 || windowStartHour >= windowEndHour) { http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanJobHelper.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanJobHelper.java b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanJobHelper.java index a74a6ad..4fbe32b 100644 --- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanJobHelper.java +++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanJobHelper.java @@ -263,7 +263,13 @@ public class AzkabanJobHelper { // Determine final zip file path String zipFilePath = String.format("%s/%s", directory, zipFilename); File zipFile = new File(zipFilePath); - zipFile.delete(); + if (zipFile.exists()) { + if (zipFile.delete()) { + log.info("Zipfile existed and was deleted: " + zipFilePath); + } else { + log.warn("Zipfile exists but was not deleted: " + zipFilePath); + } + } // Create and add files to zip file addFilesToZip(zipFile, filesToAdd); @@ -271,6 +277,9 @@ public class AzkabanJobHelper { return zipFilePath; } + @edu.umd.cs.findbugs.annotations.SuppressWarnings( + value = "OBL_UNSATISFIED_OBLIGATION", + justification = "Lombok construct of @Cleanup is handing this, but not detected by FindBugs") private static void addFilesToZip(File zipFile, List<File> filesToAdd) throws IOException { try { @Cleanup @@ -300,7 +309,13 @@ public class AzkabanJobHelper { // Determine final config file path String jobFilePath = String.format("%s/%s.job", workDir, flowName); File jobFile = new File(jobFilePath); - jobFile.delete(); + if (jobFile.exists()) { + if (jobFile.delete()) { + log.info("JobFile existed and was deleted: " + jobFilePath); + } else { + log.warn("JobFile exists but was not deleted: " + jobFilePath); + } + } StringBuilder propertyFileContent = new StringBuilder(); for (Map.Entry entry : azkabanProjectConfig.getJobSpec().getConfigAsProperties().entrySet()) { @@ -313,6 +328,9 @@ public class AzkabanJobHelper { return new File[] {jobFile}; } + @edu.umd.cs.findbugs.annotations.SuppressWarnings( + value = "OBL_UNSATISFIED_OBLIGATION", + justification = "Lombok construct of @Cleanup is handing this, but not detected by FindBugs") private static File downloadAzkabanJobJar(String workDir, String jobJarUrl) throws IOException { // Determine final jar file path @@ -320,7 +338,13 @@ public class AzkabanJobHelper { String jobJarName = jobJarUrlParts[jobJarUrlParts.length-1]; String jobJarFilePath = String.format("%s/%s", workDir, jobJarName); File jobJarFile = new File(jobJarFilePath); - jobJarFile.delete(); + if (jobJarFile.exists()) { + if (jobJarFile.delete()) { + log.info("JobJarFilePath existed and was deleted: " + jobJarFilePath); + } else { + log.warn("JobJarFilePath exists but was not deleted: " + jobJarFilePath); + } + } // Create work directory if not already exists FileUtils.forceMkdir(new File(workDir)); http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java index b99683d..583988b 100644 --- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java +++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanProjectConfig.java @@ -39,8 +39,6 @@ import com.typesafe.config.ConfigFactory; * Class to hold Azkaban project specific configs */ public class AzkabanProjectConfig { - private static final String DEFAULT_AZKABAN_PROJECT_CONFIG_FILE = "default-service-azkaban.conf"; - private final String azkabanServerUrl; private final String azkabanProjectName; @@ -60,7 +58,7 @@ public class AzkabanProjectConfig { public AzkabanProjectConfig(JobSpec jobSpec) { // Extract config objects this.jobSpec = jobSpec; - Config defaultConfig = ConfigFactory.load(DEFAULT_AZKABAN_PROJECT_CONFIG_FILE); + Config defaultConfig = ConfigFactory.load(ServiceAzkabanConfigKeys.DEFAULT_AZKABAN_PROJECT_CONFIG_FILE); Config config = jobSpec.getConfig().withFallback(defaultConfig); // Azkaban Infrastructure http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstance.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstance.java b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstance.java index 65209c3..dcc89cc 100644 --- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstance.java +++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstance.java @@ -35,6 +35,7 @@ import com.google.common.base.Splitter; import com.google.common.collect.Maps; import com.google.common.util.concurrent.AbstractIdleService; import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; public class AzkabanSpecExecutorInstance extends AbstractIdleService implements SpecExecutorInstance { @@ -48,7 +49,8 @@ public class AzkabanSpecExecutorInstance extends AbstractIdleService implements protected final Map<String, String> _capabilities; public AzkabanSpecExecutorInstance(Config config, Optional<Logger> log) { - _config = config; + Config defaultConfig = ConfigFactory.load(ServiceAzkabanConfigKeys.DEFAULT_AZKABAN_PROJECT_CONFIG_FILE); + _config = config.withFallback(defaultConfig); _log = log.isPresent() ? log.get() : LoggerFactory.getLogger(getClass()); try { _specExecutorInstanceUri = new URI(ConfigUtils.getString(config, ConfigurationKeys.SPECEXECUTOR_INSTANCE_URI_KEY, http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstanceProducer.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstanceProducer.java b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstanceProducer.java index f73bc6c..47df250 100644 --- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstanceProducer.java +++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/AzkabanSpecExecutorInstanceProducer.java @@ -47,9 +47,9 @@ public class AzkabanSpecExecutorInstanceProducer extends AzkabanSpecExecutorInst try { // Initialize Azkaban client / producer and cache credentials - String azkabanUsername = config.getString(ServiceAzkabanConfigKeys.AZKABAN_USERNAME_KEY); - String azkabanPassword = getAzkabanPassword(config); - String azkabanServerUrl = config.getString(ServiceAzkabanConfigKeys.AZKABAN_SERVER_URL_KEY); + String azkabanUsername = _config.getString(ServiceAzkabanConfigKeys.AZKABAN_USERNAME_KEY); + String azkabanPassword = getAzkabanPassword(_config); + String azkabanServerUrl = _config.getString(ServiceAzkabanConfigKeys.AZKABAN_SERVER_URL_KEY); _sessionId = AzkabanAjaxAPIClient.authenticateAndGetSessionId(azkabanUsername, azkabanPassword, azkabanServerUrl); } catch (IOException | EncoderException e) { http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/ServiceAzkabanConfigKeys.java ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/ServiceAzkabanConfigKeys.java b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/ServiceAzkabanConfigKeys.java index b712a5a..4c24944 100644 --- a/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/ServiceAzkabanConfigKeys.java +++ b/gobblin-modules/gobblin-azkaban/src/main/java/org/apache/gobblin/service/modules/orchestration/ServiceAzkabanConfigKeys.java @@ -37,5 +37,6 @@ public class ServiceAzkabanConfigKeys { // Azkaban System Environment public static final String AZKABAN_PASSWORD_SYSTEM_KEY = "GOBBLIN_SERVICE_AZKABAN_PASSWORD"; + public static final String DEFAULT_AZKABAN_PROJECT_CONFIG_FILE = "default-service-azkaban.conf"; } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-modules/gobblin-azkaban/src/main/resources/default-service-azkaban.conf ---------------------------------------------------------------------- diff --git a/gobblin-modules/gobblin-azkaban/src/main/resources/default-service-azkaban.conf b/gobblin-modules/gobblin-azkaban/src/main/resources/default-service-azkaban.conf index 6d31984..caf6ebe 100644 --- a/gobblin-modules/gobblin-azkaban/src/main/resources/default-service-azkaban.conf +++ b/gobblin-modules/gobblin-azkaban/src/main/resources/default-service-azkaban.conf @@ -37,5 +37,5 @@ gobblin.service.azkaban.project.zip.failIfJarNotFound=false gobblin.service.azkaban.project.zip.additionalFilesUrl="" gobblin.service.azkaban.project.job.jar.mavenUrlTemplate="https://repo.maven.apache.org/maven2/com/linkedin/gobblin/<module-name>/<module-version>/<module-name>-<module-version>.jar" -gobblin.service.azkaban.project.job.jar.mavenUrlTemplateModules="gobblin-admin.jar,gobblin-api.jar,gobblin-compaction.jar,gobblin-config-management.jar,gobblin-core.jar,gobblin-core-base.jar,gobblin-distribution.jar,gobblin-example.jar,gobblin-hive-registration.jar,gobblin-metrics-libs.jar,gobblin-metastore.jar,gobblin-modules.jar,gobblin-rest-service.jar,gobblin-runtime.jar,gobblin-runtime-hadoop.jar,gobblin-utility.jar,gobblin-salesforce.jar,gobblin-test-harness.jar,gobblin-tunnel.jar,gobblin-data-management.jar,gobblin-config-management.jar,gobblin-audit.jar,gobblin-yarn.jar,gobblin-cluster.jar,gobblin-aws.jar,gobblin-service.jar,gobblin-test-utils.jar" +gobblin.service.azkaban.project.job.jar.mavenUrlTemplateModules="gobblin-admin,gobblin-api,gobblin-compaction,gobblin-config-management,gobblin-core,gobblin-core-base,gobblin-distribution,gobblin-example,gobblin-hive-registration,gobblin-metrics-libs,gobblin-metastore,gobblin-modules,gobblin-rest-service,gobblin-runtime,gobblin-runtime-hadoop,gobblin-utility,gobblin-salesforce,gobblin-test-harness,gobblin-tunnel,gobblin-data-management,gobblin-config-management,gobblin-audit,gobblin-yarn,gobblin-cluster,gobblin-aws,gobblin-service,gobblin-test-utils" gobblin.service.azkaban.project.job.jar.mavenGobblinVersion="0.11.0" \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-service/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java ---------------------------------------------------------------------- diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java b/gobblin-service/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java index a6f0199..8ea19c4 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/ServiceConfigKeys.java @@ -75,4 +75,6 @@ public class ServiceConfigKeys { // Template Catalog Keys public static final String TEMPLATE_CATALOGS_FULLY_QUALIFIED_PATH_KEY = GOBBLIN_SERVICE_PREFIX + "templateCatalogs.fullyQualifiedPath"; + // Logging + public static final String GOBBLIN_SERVICE_LOG4J_CONFIGURATION_FILE = "log4j-service.properties"; } http://git-wip-us.apache.org/repos/asf/incubator-gobblin/blob/b0c96aca/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java ---------------------------------------------------------------------- diff --git a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java index 4707361..c2591e1 100644 --- a/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java +++ b/gobblin-service/src/main/java/org/apache/gobblin/service/modules/core/GobblinServiceManager.java @@ -37,6 +37,7 @@ import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.lang3.reflect.ConstructorUtils; +import org.apache.gobblin.util.logs.Log4jConfigurationHelper; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path;
