Added job count validation enable/disable capability
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/a78137b7 Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/a78137b7 Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/a78137b7 Branch: refs/heads/orchestratorJobThrottleFeature Commit: a78137b789e4573e57fd614ea015fd18ca43ecbb Parents: d899c1c Author: shamrath <[email protected]> Authored: Thu Sep 18 14:55:04 2014 -0400 Committer: shamrath <[email protected]> Committed: Thu Sep 18 14:55:04 2014 -0400 ---------------------------------------------------------------------- .../airavata/common/utils/ServerSettings.java | 5 ++++ .../main/resources/airavata-server.properties | 3 ++- .../handlers/GridPullMonitorHandler.java | 4 ++- .../monitor/impl/pull/qstat/HPCPullMonitor.java | 20 +++++++------- .../core/validator/impl/JobCountValidator.java | 28 +++++++++++++++----- 5 files changed, 42 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/a78137b7/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java ---------------------------------------------------------------------- diff --git a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java index e607cdc..6c7bb3b 100644 --- a/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java +++ b/modules/commons/utils/src/main/java/org/apache/airavata/common/utils/ServerSettings.java @@ -71,6 +71,7 @@ public class ServerSettings extends ApplicationSettings { private static final String MY_PROXY_LIFETIME = "myproxy.life"; private static final String ACTIVITY_LISTENERS = "activity.listeners"; + private static final String ENABLE_JOB_RESTRICTION_VALIDATION = "enable.job.restriction.validation"; private static final String GLOBAL_MAX_JOB_COUNT = "global.max.job.count"; private static boolean stopAllThreads = false; @@ -107,6 +108,10 @@ public class ServerSettings extends ApplicationSettings { return getSetting(GLOBAL_MAX_JOB_COUNT); } + public static String getEnableJobRestrictionValidation() { + return getSetting(ENABLE_JOB_RESTRICTION_VALIDATION, "false"); + } + public static String getCredentialStoreDBUser() throws ApplicationSettingsException { try { return getSetting(CREDENTIAL_STORE_DB_USER); http://git-wip-us.apache.org/repos/asf/airavata/blob/a78137b7/modules/configuration/server/src/main/resources/airavata-server.properties ---------------------------------------------------------------------- diff --git a/modules/configuration/server/src/main/resources/airavata-server.properties b/modules/configuration/server/src/main/resources/airavata-server.properties index cd4a2b4..42b1bc8 100644 --- a/modules/configuration/server/src/main/resources/airavata-server.properties +++ b/modules/configuration/server/src/main/resources/airavata-server.properties @@ -193,12 +193,13 @@ activity.listeners=org.apache.airavata.gfac.core.monitor.AiravataJobStatusUpdato ###---------------------------Orchestrator module Configurations---------------------------### #job.submitter=org.apache.airavata.orchestrator.core.impl.GFACEmbeddedJobSubmitter job.submitter=org.apache.airavata.orchestrator.core.impl.GFACServiceJobSubmitter -job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator,org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator +job.validators=org.apache.airavata.orchestrator.core.validator.impl.SimpleAppDataValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator,org.apache.airavata.orchestrator.core.validator.impl.JobCountValidator submitter.interval=10000 threadpool.size=10 start.submitter=true embedded.mode=true enable.validation=true +enable.job.restriction.validation=true global.max.job.count=50 orchestrator=org.apache.airavata.orchestrator.server.OrchestratorServer http://git-wip-us.apache.org/repos/asf/airavata/blob/a78137b7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java index 6428112..451466d 100644 --- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java +++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/handlers/GridPullMonitorHandler.java @@ -100,7 +100,9 @@ public class GridPullMonitorHandler extends ThreadedHandler implements Watcher{ e.printStackTrace(); } CommonUtils.addMonitortoQueue(hpcPullMonitor.getQueue(), monitorID); - CommonUtils.increaseZkJobCount(monitorID); // update change job count to zookeeper + if (ServerSettings.getEnableJobRestrictionValidation().equals("true")) { + CommonUtils.increaseZkJobCount(monitorID); // update change job count to zookeeper + } } catch (AiravataMonitorException e) { logger.error("Error adding monitorID object to the queue with experiment ", monitorID.getExperimentID()); } http://git-wip-us.apache.org/repos/asf/airavata/blob/a78137b7/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java index b0fa11a..25a1ab2 100644 --- a/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java +++ b/modules/gfac/gfac-monitor/src/main/java/org/apache/airavata/gfac/monitor/impl/pull/qstat/HPCPullMonitor.java @@ -272,17 +272,19 @@ public class HPCPullMonitor extends PullMonitor { ZooKeeper zk = null; for (MonitorID completedJob : completedJobs) { CommonUtils.removeMonitorFromQueue(queue, completedJob); - if (zk == null) { - zk = completedJob.getJobExecutionContext().getZk(); - } - String key = CommonUtils.getJobCountUpdatePath(completedJob); - int i = 0; - if (jobRemoveCountMap.containsKey(key)) { - i = Integer.valueOf(jobRemoveCountMap.get(key)); + if (ServerSettings.getEnableJobRestrictionValidation().equals("true")) { // is job restriction available? + if (zk == null) { + zk = completedJob.getJobExecutionContext().getZk(); + } + String key = CommonUtils.getJobCountUpdatePath(completedJob); + int i = 0; + if (jobRemoveCountMap.containsKey(key)) { + i = Integer.valueOf(jobRemoveCountMap.get(key)); + } + jobRemoveCountMap.put(key, ++i); } - jobRemoveCountMap.put(key, ++i); } - if (completedJobs.size() > 0) { + if (ServerSettings.getEnableJobRestrictionValidation().equals("true") && completedJobs.size() > 0) { // reduce completed job count from zookeeper CommonUtils.updateZkWithJobCount(zk, jobRemoveCountMap, false); } http://git-wip-us.apache.org/repos/asf/airavata/blob/a78137b7/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java ---------------------------------------------------------------------- diff --git a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java index 37890ec..2c66fa2 100644 --- a/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java +++ b/modules/orchestrator/orchestrator-core/src/main/java/org/apache/airavata/orchestrator/core/validator/impl/JobCountValidator.java @@ -1,3 +1,24 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * +*/ + package org.apache.airavata.orchestrator.core.validator.impl; import org.airavata.appcatalog.cpi.AppCatalog; @@ -10,11 +31,6 @@ import org.apache.airavata.common.utils.AiravataUtils; import org.apache.airavata.common.utils.Constants; import org.apache.airavata.common.utils.RequestData; import org.apache.airavata.common.utils.ServerSettings; -import org.apache.airavata.credential.store.credential.AuditInfo; -import org.apache.airavata.credential.store.store.CredentialReader; -import org.apache.airavata.credential.store.store.CredentialReaderFactory; -import org.apache.airavata.credential.store.store.CredentialStoreException; -import org.apache.airavata.credential.store.util.TokenizedMyProxyAuthInfo; import org.apache.airavata.gfac.core.scheduler.HostScheduler; import org.apache.airavata.model.appcatalog.appdeployment.ApplicationDeploymentDescription; import org.apache.airavata.model.appcatalog.appinterface.ApplicationInterfaceDescription; @@ -28,12 +44,10 @@ import org.apache.airavata.model.workspace.experiment.WorkflowNodeDetails; import org.apache.airavata.orchestrator.core.context.OrchestratorContext; import org.apache.airavata.orchestrator.core.validator.JobMetadataValidator; import org.apache.airavata.persistance.registry.jpa.model.TaskDetail; -import org.apache.airavata.registry.cpi.RegistryException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map;
