This is an automated email from the ASF dual-hosted git repository. yasith pushed a commit to branch service-layer-improvements in repository https://gitbox.apache.org/repos/asf/airavata.git
commit 0fda5849d5d2e8f47c44c0b66770fb98ec588ba3 Author: yasithdev <[email protected]> AuthorDate: Sun Dec 14 03:34:02 2025 -0600 migrate commons library usages to spring, flatten model/util to util --- airavata-api/pom.xml | 35 +--- .../airavata/api/thrift/util/ThriftClientPool.java | 229 +++++++++++---------- .../airavata/common/utils/ApplicationSettings.java | 21 +- .../org/apache/airavata/common/utils/JPAUtils.java | 8 +- .../org/apache/airavata/common/utils/JSONUtil.java | 4 +- .../apache/airavata/common/utils/StringUtil.java | 85 +++----- .../credential/impl/notifier/EmailNotifier.java | 44 ++-- .../apache/airavata/credential/utils/Utility.java | 6 +- .../helix/agent/ssh/StandardOutReader.java | 20 +- .../airavata/helix/impl/task/AiravataTask.java | 39 ++-- .../airavata/helix/impl/task/TaskContext.java | 2 +- .../helix/impl/task/aws/AWSJobSubmissionTask.java | 10 +- .../helix/impl/task/parsing/DataParsingTask.java | 6 +- .../helix/impl/task/staging/DataStagingTask.java | 4 +- .../impl/task/submission/JobSubmissionTask.java | 6 +- .../impl/task/submission/config/GroovyMapData.java | 5 +- .../config/app/CloudJobManagerConfiguration.java | 6 +- .../config/app/ForkJobConfiguration.java | 4 +- .../config/app/HTCondorJobConfiguration.java | 4 +- .../submission/config/app/LSFJobConfiguration.java | 4 +- .../submission/config/app/PBSJobConfiguration.java | 4 +- .../config/app/SlurmJobConfiguration.java | 4 +- .../submission/config/app/UGEJobConfiguration.java | 6 +- .../messaging/client/RabbitMQListener.java | 87 ++++---- .../metadata/analyzer/DataInterpreterService.java | 2 +- .../rescheduler/ProcessReschedulingService.java | 2 +- .../cluster/ClusterStatusMonitorJobScheduler.java | 2 +- .../ComputationalResourceMonitoringService.java | 2 +- .../orchestrator/impl/SimpleOrchestratorImpl.java | 5 +- .../utils/migration/MappingToolRunner.java | 14 +- .../airavata/security/KeyCloakSecurityManager.java | 34 +-- .../security/interceptor/SecurityModule.java | 3 +- .../airavata/security/userstore/JDBCUserStore.java | 31 +-- .../airavata/security/userstore/LDAPUserStore.java | 22 +- .../service/orchestrator/OrchestratorService.java | 5 +- .../service/security/CredentialStoreService.java | 6 +- .../{model => }/util/AppDeploymentUtil.java | 2 +- .../{model => }/util/AppInterfaceUtil.java | 2 +- .../airavata/{model => }/util/ExecutionType.java | 2 +- .../{model => }/util/ExperimentModelUtil.java | 2 +- .../util/GroupComputeResourcePreferenceUtil.java | 2 +- .../{model => }/util/ProjectModelUtil.java | 2 +- .../airavata/api/thrift/client/TestSSLClient.java | 6 +- .../server/TestOrchestratorServiceServer.java | 2 +- .../api/thrift/util/ThriftClientPoolTest.java | 180 ++++------------ .../airavata/config/SchemaValidationTest.java | 59 +++--- .../security/KeyCloakSecurityManagerTest.java | 10 +- .../GroupComputeResourcePreferenceUtilTest.java | 3 +- .../service/db/entity/AgentExecutionStatus.java | 6 +- .../service/handlers/AgentConnectionHandler.java | 1 - .../connection/service/handlers/FuseFSHandler.java | 22 +- .../apache/airavata/registry/tool/DBMigrator.java | 3 +- 52 files changed, 496 insertions(+), 579 deletions(-) diff --git a/airavata-api/pom.xml b/airavata-api/pom.xml index 505e616a82..6f98e9c6e2 100644 --- a/airavata-api/pom.xml +++ b/airavata-api/pom.xml @@ -102,10 +102,6 @@ under the License. </dependency> <!-- CLI & mapping --> - <dependency> - <groupId>commons-cli</groupId> - <artifactId>commons-cli</artifactId> - </dependency> <!-- Dozer - being migrated to MapStruct --> <dependency> <groupId>com.github.dozermapper</groupId> @@ -150,22 +146,6 @@ under the License. <groupId>com.github.mwiede</groupId> <artifactId>jsch</artifactId> </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-email</artifactId> - </dependency> - <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - <dependency> - <groupId>commons-codec</groupId> - <artifactId>commons-codec</artifactId> - </dependency> - <dependency> - <groupId>org.json</groupId> - <artifactId>json</artifactId> - </dependency> <!-- Test dependencies --> <dependency> @@ -208,6 +188,11 @@ under the License. <groupId>org.springframework.security</groupId> <artifactId>spring-security-ldap</artifactId> </dependency> + <!-- Spring Mail for email notifications --> + <dependency> + <groupId>org.springframework.boot</groupId> + <artifactId>spring-boot-starter-mail</artifactId> + </dependency> <dependency> <groupId>io.prometheus</groupId> @@ -323,10 +308,6 @@ under the License. <groupId>org.databene</groupId> <artifactId>contiperf</artifactId> </dependency> - <dependency> - <groupId>org.apache.commons</groupId> - <artifactId>commons-lang3</artifactId> - </dependency> <dependency> <groupId>software.amazon.awssdk</groupId> @@ -372,6 +353,12 @@ under the License. <dependency> <groupId>org.springframework.boot</groupId> <artifactId>spring-boot-starter-data-jpa</artifactId> + <exclusions> + <exclusion> + <groupId>io.smallrye</groupId> + <artifactId>jandex</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.springframework.boot</groupId> diff --git a/airavata-api/src/main/java/org/apache/airavata/api/thrift/util/ThriftClientPool.java b/airavata-api/src/main/java/org/apache/airavata/api/thrift/util/ThriftClientPool.java index 67defb550c..e17a9382ac 100644 --- a/airavata-api/src/main/java/org/apache/airavata/api/thrift/util/ThriftClientPool.java +++ b/airavata-api/src/main/java/org/apache/airavata/api/thrift/util/ThriftClientPool.java @@ -19,16 +19,12 @@ */ package org.apache.airavata.api.thrift.util; -import java.io.PrintWriter; -import java.io.StringWriter; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.airavata.api.model.BaseAPI; import org.apache.airavata.config.AiravataServerProperties; -import org.apache.commons.pool2.BasePooledObjectFactory; -import org.apache.commons.pool2.PooledObject; -import org.apache.commons.pool2.impl.AbandonedConfig; -import org.apache.commons.pool2.impl.DefaultPooledObject; -import org.apache.commons.pool2.impl.GenericObjectPool; -import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.thrift.protocol.TBinaryProtocol; import org.apache.thrift.protocol.TProtocol; import org.apache.thrift.transport.TSocket; @@ -41,119 +37,61 @@ public class ThriftClientPool<T extends BaseAPI.Client> implements AutoCloseable private static final Logger logger = LoggerFactory.getLogger(ThriftClientPool.class); - private final GenericObjectPool<T> internalPool; + private final BlockingQueue<T> pool; + private final ClientFactory<T> clientFactory; + private final ProtocolFactory protocolFactory; + private final int maxTotal; + private final int maxIdle; + private final long maxWaitMillis; + private final long timeBetweenEvictionRunsMillis; + private final AtomicInteger createdCount = new AtomicInteger(0); private final AiravataServerProperties properties; - /** - * StringWriter that flushes to SLF4J logger. - */ - private static class ErrorLoggingStringWriter extends StringWriter { - - @Override - public void flush() { - logger.error(this.toString()); - // Reset buffer - this.getBuffer().setLength(0); - } + public ThriftClientPool(ClientFactory<T> clientFactory, PoolConfig poolConfig, String host, int port) { + this(clientFactory, new BinaryOverSocketProtocolFactory(host, port), poolConfig, null); } - public ThriftClientPool( - ClientFactory<T> clientFactory, GenericObjectPoolConfig<T> poolConfig, String host, int port) { - this(clientFactory, new BinaryOverSocketProtocolFactory(host, port), poolConfig, null, null); - } - - public ThriftClientPool( - ClientFactory<T> clientFactory, ProtocolFactory protocolFactory, GenericObjectPoolConfig<T> poolConfig) { - this(clientFactory, protocolFactory, poolConfig, null, null); + public ThriftClientPool(ClientFactory<T> clientFactory, ProtocolFactory protocolFactory, PoolConfig poolConfig) { + this(clientFactory, protocolFactory, poolConfig, null); } public ThriftClientPool( ClientFactory<T> clientFactory, ProtocolFactory protocolFactory, - GenericObjectPoolConfig<T> poolConfig, - AiravataServerProperties properties) { - this(clientFactory, protocolFactory, poolConfig, null, properties); - } - - public ThriftClientPool( - ClientFactory<T> clientFactory, - ProtocolFactory protocolFactory, - GenericObjectPoolConfig<T> poolConfig, - AbandonedConfig abandonedConfig) { - this(clientFactory, protocolFactory, poolConfig, abandonedConfig, null); - } - - private ThriftClientPool( - ClientFactory<T> clientFactory, - ProtocolFactory protocolFactory, - GenericObjectPoolConfig<T> poolConfig, - AbandonedConfig abandonedConfig, + PoolConfig poolConfig, AiravataServerProperties properties) { + this.clientFactory = clientFactory; + this.protocolFactory = protocolFactory; this.properties = properties; - - // If abandonedConfig not provided but properties are, create from properties - if (abandonedConfig == null && properties != null) { - if (properties.airavata.thriftClientPoolAbandonedRemovalEnabled) { - abandonedConfig = new AbandonedConfig(); - abandonedConfig.setRemoveAbandonedOnBorrow(true); - abandonedConfig.setRemoveAbandonedOnMaintenance(true); - if (properties.airavata.thriftClientPoolAbandonedRemovalLogged) { - abandonedConfig.setLogAbandoned(true); - abandonedConfig.setLogWriter(new PrintWriter(new ErrorLoggingStringWriter())); - } else { - abandonedConfig.setLogAbandoned(false); - } - } - } - - if (abandonedConfig != null - && abandonedConfig.getRemoveAbandonedOnMaintenance() - && poolConfig.getTimeBetweenEvictionRunsMillis() <= 0) { - logger.warn( - "Abandoned removal is enabled but" - + " removeAbandonedOnMaintenance won't run since" - + " timeBetweenEvictionRunsMillis is not positive, current value: {}", - poolConfig.getTimeBetweenEvictionRunsMillis()); - } - this.internalPool = new GenericObjectPool<T>( - new ThriftClientFactory(clientFactory, protocolFactory), poolConfig, abandonedConfig); + this.maxTotal = poolConfig.getMaxTotal(); + this.maxIdle = poolConfig.getMaxIdle() > 0 ? poolConfig.getMaxIdle() : poolConfig.getMaxTotal(); + this.maxWaitMillis = poolConfig.getMaxWaitMillis(); + this.timeBetweenEvictionRunsMillis = poolConfig.getTimeBetweenEvictionRunsMillis(); + this.pool = new LinkedBlockingQueue<>(maxIdle); } - class ThriftClientFactory extends BasePooledObjectFactory<T> { - - private ClientFactory<T> clientFactory; - private ProtocolFactory protocolFactory; - - public ThriftClientFactory(ClientFactory<T> clientFactory, ProtocolFactory protocolFactory) { - this.clientFactory = clientFactory; - this.protocolFactory = protocolFactory; - } - - @Override - public T create() throws Exception { - try { - TProtocol protocol = protocolFactory.make(); - return clientFactory.make(protocol); - } catch (Exception e) { - logger.warn(e.getMessage(), e); - throw new ThriftClientException("Can not make a new object for pool", e); - } + private T createClient() throws Exception { + try { + TProtocol protocol = protocolFactory.make(); + T client = clientFactory.make(protocol); + createdCount.incrementAndGet(); + return client; + } catch (Exception e) { + logger.warn(e.getMessage(), e); + throw new ThriftClientException("Can not make a new object for pool", e); } + } - @Override - public void destroyObject(PooledObject<T> pooledObject) throws Exception { - T obj = pooledObject.getObject(); - if (obj.getOutputProtocol().getTransport().isOpen()) { - obj.getOutputProtocol().getTransport().close(); + private void destroyClient(T client) { + try { + if (client.getOutputProtocol().getTransport().isOpen()) { + client.getOutputProtocol().getTransport().close(); } - if (obj.getInputProtocol().getTransport().isOpen()) { - obj.getInputProtocol().getTransport().close(); + if (client.getInputProtocol().getTransport().isOpen()) { + client.getInputProtocol().getTransport().close(); } - } - - @Override - public PooledObject<T> wrap(T obj) { - return new DefaultPooledObject<T>(obj); + } catch (Exception e) { + logger.warn("Error destroying client", e); } } @@ -204,17 +142,29 @@ public class ThriftClientPool<T extends BaseAPI.Client> implements AutoCloseable try { for (int i = 0; i < 10; i++) { // This tries to fetch a client from the pool and validate it before returning. - final T client = internalPool.borrowObject(); + T client = null; try { + // Try to get from pool first + client = pool.poll(maxWaitMillis, TimeUnit.MILLISECONDS); + if (client == null && createdCount.get() < maxTotal) { + // Create new client if under max total + client = createClient(); + } + if (client == null) { + throw new Exception("Pool exhausted and max total reached"); + } + // Validate client String apiVersion = client.getAPIVersion(); logger.debug("Validated client and fetched api version " + apiVersion); return client; } catch (Exception e) { logger.warn("Failed to validate the client. Retrying " + i, e); - returnBrokenResource(client); + if (client != null) { + returnBrokenResource(client); + } } } - throw new Exception("Failed to fetch a client form the pool after validation"); + throw new Exception("Failed to fetch a client from the pool after validation"); } catch (Exception e) { throw new ThriftClientException("Could not get a resource from the pool", e); } @@ -222,9 +172,15 @@ public class ThriftClientPool<T extends BaseAPI.Client> implements AutoCloseable public void returnResourceObject(T resource) { try { - internalPool.returnObject(resource); + if (!pool.offer(resource)) { + // Pool is full, destroy the client + destroyClient(resource); + createdCount.decrementAndGet(); + } } catch (Exception e) { - throw new ThriftClientException("Could not return the resource to the pool", e); + logger.warn("Error returning resource to pool", e); + destroyClient(resource); + createdCount.decrementAndGet(); } } @@ -238,9 +194,10 @@ public class ThriftClientPool<T extends BaseAPI.Client> implements AutoCloseable protected void returnBrokenResourceObject(T resource) { try { - internalPool.invalidateObject(resource); + destroyClient(resource); + createdCount.decrementAndGet(); } catch (Exception e) { - throw new ThriftClientException("Could not return the resource to the pool", e); + logger.warn("Error destroying broken resource", e); } } @@ -250,9 +207,55 @@ public class ThriftClientPool<T extends BaseAPI.Client> implements AutoCloseable public void close() { try { - internalPool.close(); + T client; + while ((client = pool.poll()) != null) { + destroyClient(client); + createdCount.decrementAndGet(); + } } catch (Exception e) { - throw new ThriftClientException("Could not destroy the pool", e); + logger.error("Error closing pool", e); + } + } + + /** + * Simple pool configuration class to replace GenericObjectPoolConfig + */ + public static class PoolConfig { + private int maxTotal = 8; + private int maxIdle = 8; + private long maxWaitMillis = -1; + private long timeBetweenEvictionRunsMillis = -1; + + public int getMaxTotal() { + return maxTotal; + } + + public void setMaxTotal(int maxTotal) { + this.maxTotal = maxTotal; + } + + public int getMaxIdle() { + return maxIdle; + } + + public void setMaxIdle(int maxIdle) { + this.maxIdle = maxIdle; + } + + public long getMaxWaitMillis() { + return maxWaitMillis; + } + + public void setMaxWaitMillis(long maxWaitMillis) { + this.maxWaitMillis = maxWaitMillis; + } + + public long getTimeBetweenEvictionRunsMillis() { + return timeBetweenEvictionRunsMillis; + } + + public void setTimeBetweenEvictionRunsMillis(long timeBetweenEvictionRunsMillis) { + this.timeBetweenEvictionRunsMillis = timeBetweenEvictionRunsMillis; } } } diff --git a/airavata-api/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java b/airavata-api/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java index b570c3f8eb..a24b87af96 100644 --- a/airavata-api/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java +++ b/airavata-api/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java @@ -29,11 +29,9 @@ import java.util.ArrayList; import java.util.Calendar; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Properties; import java.util.regex.Pattern; import org.apache.airavata.common.exception.ApplicationSettingsException; -import org.apache.commons.lang3.BooleanUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -227,8 +225,23 @@ public class ApplicationSettings { public static boolean getBooleanSetting(String key) throws ApplicationSettingsException { String val = getInstance().getSettingImpl(key); - return Optional.ofNullable(BooleanUtils.toBooleanObject(val)) - .orElseThrow(() -> new ApplicationSettingsException("Value can not be parsed to Boolean")); + if (val == null) { + throw new ApplicationSettingsException("Value can not be parsed to Boolean"); + } + String normalized = val.trim().toLowerCase(); + if ("true".equals(normalized) + || "yes".equals(normalized) + || "on".equals(normalized) + || "1".equals(normalized)) { + return true; + } else if ("false".equals(normalized) + || "no".equals(normalized) + || "off".equals(normalized) + || "0".equals(normalized)) { + return false; + } else { + throw new ApplicationSettingsException("Value can not be parsed to Boolean"); + } } public static boolean isSettingDefined(String key) throws ApplicationSettingsException { diff --git a/airavata-api/src/main/java/org/apache/airavata/common/utils/JPAUtils.java b/airavata-api/src/main/java/org/apache/airavata/common/utils/JPAUtils.java index 9a99e6f51e..052a6e5419 100644 --- a/airavata-api/src/main/java/org/apache/airavata/common/utils/JPAUtils.java +++ b/airavata-api/src/main/java/org/apache/airavata/common/utils/JPAUtils.java @@ -21,7 +21,6 @@ package org.apache.airavata.common.utils; import jakarta.persistence.EntityManagerFactory; import jakarta.persistence.Persistence; -import java.util.Collections; import java.util.HashMap; import java.util.Map; import org.slf4j.Logger; @@ -92,8 +91,11 @@ public class JPAUtils { properties.put("jakarta.persistence.jdbc.url", url + urlSuffix); properties.put("jakarta.persistence.jdbc.user", jdbcConfig.getUser()); properties.put("jakarta.persistence.jdbc.password", jdbcConfig.getPassword()); - logger.debug("Connection properties: driver={}, url={}, user={}", - jdbcConfig.getDriver(), url + urlSuffix, jdbcConfig.getUser()); + logger.debug( + "Connection properties: driver={}, url={}, user={}", + jdbcConfig.getDriver(), + url + urlSuffix, + jdbcConfig.getUser()); return properties; } diff --git a/airavata-api/src/main/java/org/apache/airavata/common/utils/JSONUtil.java b/airavata-api/src/main/java/org/apache/airavata/common/utils/JSONUtil.java index 1223a0b726..eafbddbbe5 100644 --- a/airavata-api/src/main/java/org/apache/airavata/common/utils/JSONUtil.java +++ b/airavata-api/src/main/java/org/apache/airavata/common/utils/JSONUtil.java @@ -90,11 +90,11 @@ public class JSONUtil { String key = fieldNames.next(); JsonNode valueOrig = originalJsonObject.get(key); JsonNode valueNew = newJsonObject.get(key); - + if (valueNew == null) { return false; } - + if (valueOrig.isObject() && valueNew.isObject()) { if (!isEqual((ObjectNode) valueOrig, (ObjectNode) valueNew)) { return false; diff --git a/airavata-api/src/main/java/org/apache/airavata/common/utils/StringUtil.java b/airavata-api/src/main/java/org/apache/airavata/common/utils/StringUtil.java index 51253ed597..7891ae611f 100644 --- a/airavata-api/src/main/java/org/apache/airavata/common/utils/StringUtil.java +++ b/airavata-api/src/main/java/org/apache/airavata/common/utils/StringUtil.java @@ -25,16 +25,9 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; -import java.util.ListIterator; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.PosixParser; public class StringUtil { public static final String DELIMETER = ","; @@ -375,31 +368,6 @@ public class StringUtil { return byteArrayOutputStream.toString(); } - private static Options deriveCommandLineOptions(String[] args) { - Options options = new Options(); - String[] argCopy = getChangedList(args); - int i = 0; - for (String arg : argCopy) { - if (arg.startsWith("--")) { - arg = arg.substring(2); - int pos = arg.indexOf('='); - String opt; - boolean hasArgs = true; - if (pos == -1) { // if not of the form --arg=value - if (i == argCopy.length - 1 || argCopy[i + 1].startsWith("-")) { // no value specified - hasArgs = false; - } - opt = arg; - } else { - opt = arg.substring(0, pos); - } - options.addOption(opt, hasArgs, ""); - } - i++; - } - return options; - } - public static Map<String, String> parseCommandLineOptions(String[] args) { Map<String, String> commandLineOptions = new HashMap<String, String>(); try { @@ -408,17 +376,40 @@ public class StringUtil { for (String s : parameters.keySet()) { commandLineOptions.put(s, parameters.get(s) == null ? "" : parameters.get(s)); } - } catch (ParseException e1) { + } catch (Exception e1) { e1.printStackTrace(); } return commandLineOptions; } - public static CommandLineParameters getCommandLineParser(String[] args) throws ParseException { + public static CommandLineParameters getCommandLineParser(String[] args) throws Exception { String[] argCopy = getChangedList(args); - CommandLineParser parser = new DynamicOptionPosixParser(); - CommandLine cmdLine = parser.parse(deriveCommandLineOptions(argCopy), argCopy); - return new CommandLineParameters(cmdLine); + Map<String, String> options = new HashMap<>(); + List<String> arguments = new ArrayList<>(); + + for (int i = 0; i < argCopy.length; i++) { + String arg = argCopy[i]; + if (arg.startsWith("--")) { + arg = arg.substring(2); + int pos = arg.indexOf('='); + String opt; + String value = null; + if (pos == -1) { // if not of the form --arg=value + opt = arg; + if (i + 1 < argCopy.length && !argCopy[i + 1].startsWith("-")) { + value = argCopy[++i]; + } + } else { + opt = arg.substring(0, pos); + value = arg.substring(pos + 1); + } + options.put(revertOption(opt), value != null ? revertOption(value) : ""); + } else if (!arg.startsWith("-")) { + arguments.add(revertOption(arg)); + } + } + + return new CommandLineParameters(options, arguments); } // commons-cli does not support arg names having the period (".") @@ -440,27 +431,13 @@ public class StringUtil { return option == null ? option : option.replaceAll(Pattern.quote("."), ARG_DOT_REPLACE); } - private static class DynamicOptionPosixParser extends PosixParser { - @Override - protected void processOption(String arg0, @SuppressWarnings("rawtypes") ListIterator arg1) - throws ParseException { - if (getOptions().hasOption(arg0)) { - super.processOption(arg0, arg1); - } - } - } - public static class CommandLineParameters { private Map<String, String> parameters = new HashMap<String, String>(); private List<String> arguments = new ArrayList<String>(); - protected CommandLineParameters(CommandLine cmd) { - for (Option opt : cmd.getOptions()) { - parameters.put(revertOption(opt.getOpt()), revertOption(opt.getValue())); - } - for (String arg : cmd.getArgs()) { - arguments.add(revertOption(arg)); - } + protected CommandLineParameters(Map<String, String> options, List<String> args) { + this.parameters = options; + this.arguments = args; } public List<String> getArguments() { diff --git a/airavata-api/src/main/java/org/apache/airavata/credential/impl/notifier/EmailNotifier.java b/airavata-api/src/main/java/org/apache/airavata/credential/impl/notifier/EmailNotifier.java index 830f8abcca..cfde541d46 100644 --- a/airavata-api/src/main/java/org/apache/airavata/credential/impl/notifier/EmailNotifier.java +++ b/airavata-api/src/main/java/org/apache/airavata/credential/impl/notifier/EmailNotifier.java @@ -22,12 +22,11 @@ package org.apache.airavata.credential.impl.notifier; import org.apache.airavata.credential.exception.CredentialStoreException; import org.apache.airavata.credential.utils.CredentialStoreNotifier; import org.apache.airavata.credential.utils.NotificationMessage; -import org.apache.commons.mail.DefaultAuthenticator; -import org.apache.commons.mail.Email; -import org.apache.commons.mail.EmailException; -import org.apache.commons.mail.SimpleEmail; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.mail.SimpleMailMessage; +import org.springframework.mail.javamail.JavaMailSender; +import org.springframework.mail.javamail.JavaMailSenderImpl; /** * User: AmilaJ ([email protected]) @@ -46,27 +45,40 @@ public class EmailNotifier implements CredentialStoreNotifier { public void notifyMessage(NotificationMessage message) throws CredentialStoreException { try { - Email email = new SimpleEmail(); - email.setHostName(this.emailNotifierConfiguration.getEmailServer()); - email.setSmtpPort(this.emailNotifierConfiguration.getEmailServerPort()); - email.setAuthenticator(new DefaultAuthenticator( - this.emailNotifierConfiguration.getEmailUserName(), - this.emailNotifierConfiguration.getEmailPassword())); - email.setSSLOnConnect(this.emailNotifierConfiguration.isSslConnect()); + JavaMailSender mailSender = createMailSender(); + SimpleMailMessage email = new SimpleMailMessage(); email.setFrom(this.emailNotifierConfiguration.getFromAddress()); EmailNotificationMessage emailMessage = (EmailNotificationMessage) message; email.setSubject(emailMessage.getSubject()); - email.setMsg(emailMessage.getMessage()); - email.addTo(emailMessage.getSenderEmail()); - email.send(); + email.setText(emailMessage.getMessage()); + email.setTo(emailMessage.getSenderEmail()); + mailSender.send(email); - } catch (EmailException e) { - log.error("[CredentialStore]Error sending email notification message."); + } catch (Exception e) { + log.error("[CredentialStore]Error sending email notification message.", e); CredentialStoreException cse = new CredentialStoreException("Error sending email notification message"); cse.initCause(e); throw cse; } } + + private JavaMailSender createMailSender() { + JavaMailSenderImpl mailSender = new JavaMailSenderImpl(); + mailSender.setHost(this.emailNotifierConfiguration.getEmailServer()); + mailSender.setPort(this.emailNotifierConfiguration.getEmailServerPort()); + mailSender.setUsername(this.emailNotifierConfiguration.getEmailUserName()); + mailSender.setPassword(this.emailNotifierConfiguration.getEmailPassword()); + + java.util.Properties props = mailSender.getJavaMailProperties(); + props.put("mail.transport.protocol", "smtp"); + if (this.emailNotifierConfiguration.isSslConnect()) { + props.put("mail.smtp.ssl.enable", "true"); + props.put("mail.smtp.starttls.enable", "true"); + } + props.put("mail.smtp.auth", "true"); + + return mailSender; + } } diff --git a/airavata-api/src/main/java/org/apache/airavata/credential/utils/Utility.java b/airavata-api/src/main/java/org/apache/airavata/credential/utils/Utility.java index b964c81d52..2a900bf530 100644 --- a/airavata-api/src/main/java/org/apache/airavata/credential/utils/Utility.java +++ b/airavata-api/src/main/java/org/apache/airavata/credential/utils/Utility.java @@ -23,13 +23,13 @@ import com.jcraft.jsch.JSch; import com.jcraft.jsch.KeyPair; import java.io.File; import java.io.FileInputStream; +import java.nio.file.Files; import java.security.KeyStore; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; import org.apache.airavata.credential.impl.ssh.SSHCredential; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -91,9 +91,9 @@ public class Utility { kpair.writePrivateKey(fileName, credential.getPassphrase().getBytes()); kpair.writePublicKey(fileName + ".pub", ""); kpair.dispose(); - byte[] priKey = FileUtils.readFileToByteArray(new File(fileName)); + byte[] priKey = Files.readAllBytes(new File(fileName).toPath()); - byte[] pubKey = FileUtils.readFileToByteArray(new File(fileName + ".pub")); + byte[] pubKey = Files.readAllBytes(new File(fileName + ".pub").toPath()); credential.setPrivateKey(priKey); credential.setPublicKey(pubKey); return credential; diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/StandardOutReader.java b/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/StandardOutReader.java index 1f79815aff..90511f9e81 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/StandardOutReader.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/agent/ssh/StandardOutReader.java @@ -19,11 +19,13 @@ */ package org.apache.airavata.helix.agent.ssh; +import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.io.StringWriter; +import java.nio.charset.StandardCharsets; import org.apache.airavata.agents.api.CommandOutput; -import org.apache.commons.io.IOUtils; /** * TODO: Class level comments please @@ -54,13 +56,25 @@ public class StandardOutReader implements CommandOutput { public void readStdOutFromStream(InputStream is) throws IOException { StringWriter writer = new StringWriter(); - IOUtils.copy(is, writer, "UTF-8"); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + writer.write(line); + writer.write(System.lineSeparator()); + } + } this.stdOut = writer.toString(); } public void readStdErrFromStream(InputStream is) throws IOException { StringWriter writer = new StringWriter(); - IOUtils.copy(is, writer, "UTF-8"); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + writer.write(line); + writer.write(System.lineSeparator()); + } + } this.stdError = writer.toString(); } diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java index 14f3f030c4..b202104367 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/AiravataTask.java @@ -19,9 +19,14 @@ */ package org.apache.airavata.helix.impl.task; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.File; import java.io.IOException; +import java.io.PrintWriter; import java.io.StringWriter; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -64,12 +69,8 @@ import org.apache.airavata.messaging.core.impl.RabbitMQPublisher; import org.apache.airavata.service.profile.UserProfileService; import org.apache.airavata.service.registry.RegistryService; import org.apache.airavata.service.security.CredentialStoreService; -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.helix.HelixManager; import org.apache.helix.task.TaskResult; -import org.json.JSONException; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.MDC; @@ -85,6 +86,7 @@ public abstract class AiravataTask extends AbstractTask { protected final RegistryService registryService; private final UserProfileService userProfileService; private final CredentialStoreService credentialStoreService; + private final ObjectMapper objectMapper = new ObjectMapper(); public AiravataTask( ApplicationContext applicationContext, @@ -195,7 +197,7 @@ public abstract class AiravataTask extends AbstractTask { error = new TaskOnFailException(errorMessage, true, error); status.setReason(errorMessage); - errors.write(ExceptionUtils.getStackTrace(error)); + error.printStackTrace(new PrintWriter(errors)); logger.error(errorMessage, error); status.setTimeOfStateChange(AiravataUtils.getCurrentTimestamp().getTime()); @@ -262,7 +264,18 @@ public abstract class AiravataTask extends AbstractTask { localDataPath = localDataPath + getProcessId(); try { - FileUtils.deleteDirectory(new File(localDataPath)); + Path path = Path.of(localDataPath); + if (Files.exists(path)) { + Files.walk(path) + .sorted((a, b) -> b.compareTo(a)) // Delete files before directories + .forEach(p -> { + try { + Files.delete(p); + } catch (IOException e) { + logger.warn("Failed to delete " + p, e); + } + }); + } } catch (IOException e) { logger.error("Failed to delete local data directory " + localDataPath, e); } @@ -422,15 +435,15 @@ public abstract class AiravataTask extends AbstractTask { // Copy experiment output's file-metadata to data product's metadata if (outputMetadata != null) { try { - JSONObject outputMetadataJSON = new JSONObject(outputMetadata); + JsonNode outputMetadataJSON = objectMapper.readTree(outputMetadata); if (outputMetadataJSON.has("file-metadata")) { - JSONObject fileMetadata = outputMetadataJSON.getJSONObject("file-metadata"); - for (Object key : fileMetadata.keySet()) { - String k = key.toString(); - dataProductModel.putToProductMetadata(k, fileMetadata.getString(k)); - } + JsonNode fileMetadata = outputMetadataJSON.get("file-metadata"); + fileMetadata.fields().forEachRemaining(entry -> { + dataProductModel.putToProductMetadata( + entry.getKey(), entry.getValue().asText()); + }); } - } catch (JSONException e) { + } catch (Exception e) { logger.warn("Failed to parse output metadata: [" + outputMetadata + "]", e); } } diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/TaskContext.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/TaskContext.java index 80f86c1512..40f70b017b 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/TaskContext.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/TaskContext.java @@ -66,12 +66,12 @@ import org.apache.airavata.common.model.UserResourceProfile; import org.apache.airavata.common.model.UserStoragePreference; import org.apache.airavata.common.utils.AiravataUtils; import org.apache.airavata.messaging.core.Publisher; -import org.apache.airavata.model.util.GroupComputeResourcePreferenceUtil; import org.apache.airavata.registry.exception.RegistryServiceException; import org.apache.airavata.security.AiravataSecurityManager; import org.apache.airavata.security.model.AuthzToken; import org.apache.airavata.service.profile.UserProfileService; import org.apache.airavata.service.registry.RegistryService; +import org.apache.airavata.util.GroupComputeResourcePreferenceUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/aws/AWSJobSubmissionTask.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/aws/AWSJobSubmissionTask.java index 90f02a6d20..51540f8702 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/aws/AWSJobSubmissionTask.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/aws/AWSJobSubmissionTask.java @@ -21,6 +21,8 @@ package org.apache.airavata.helix.impl.task.aws; import java.io.File; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; import java.security.SecureRandom; import java.util.Collections; import java.util.concurrent.TimeUnit; @@ -45,7 +47,6 @@ import org.apache.airavata.helix.task.api.TaskHelper; import org.apache.airavata.helix.task.api.annotation.TaskDef; import org.apache.airavata.registry.exception.RegistryServiceException; import org.apache.airavata.service.security.CredentialStoreService; -import org.apache.commons.io.FileUtils; import org.apache.helix.task.TaskResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -124,7 +125,12 @@ public class AWSJobSubmissionTask extends JobSubmissionTask { File localScriptFile = new File( getLocalDataDir(), "aws-job-" + new SecureRandom().nextInt() + jobManagerConfig.getScriptExtension()); - FileUtils.writeStringToFile(localScriptFile, scriptContent, StandardCharsets.UTF_8); + Files.writeString( + localScriptFile.toPath(), + scriptContent, + StandardCharsets.UTF_8, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING); jobModel.setJobStatuses(Collections.singletonList(new JobStatus(JobState.QUEUED))); saveJobModel(jobModel); diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/parsing/DataParsingTask.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/parsing/DataParsingTask.java index 3a2375d3eb..743640767f 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/parsing/DataParsingTask.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/parsing/DataParsingTask.java @@ -32,6 +32,7 @@ import com.github.dockerjava.core.command.WaitContainerResultCallback; import java.io.File; import java.io.IOException; import java.net.URI; +import java.nio.file.Files; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -63,7 +64,6 @@ import org.apache.airavata.helix.task.api.support.AdaptorSupport; import org.apache.airavata.monitor.platform.CountMonitor; import org.apache.airavata.registry.exception.RegistryServiceException; import org.apache.airavata.service.registry.RegistryService; -import org.apache.commons.io.FileUtils; import org.apache.helix.task.TaskResult; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -471,7 +471,7 @@ public class DataParsingTask extends AbstractTask { + "parsers" + File.separator + containerName + File.separator + "data" + File.separator + "input" + File.separator; try { - FileUtils.forceMkdir(new File(localInpDir)); + Files.createDirectories(new File(localInpDir).toPath()); return localInpDir; } catch (IOException e) { @@ -484,7 +484,7 @@ public class DataParsingTask extends AbstractTask { + "parsers" + File.separator + containerName + File.separator + "data" + File.separator + "output" + File.separator; try { - FileUtils.forceMkdir(new File(localOutDir)); + Files.createDirectories(new File(localOutDir).toPath()); return localOutDir; } catch (IOException e) { diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java index 367667744e..4458f73b0a 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/staging/DataStagingTask.java @@ -23,6 +23,7 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Files; import java.util.HashMap; import java.util.Map; import java.util.concurrent.Callable; @@ -47,7 +48,6 @@ import org.apache.airavata.helix.impl.task.AiravataTask; import org.apache.airavata.helix.impl.task.TaskOnFailException; import org.apache.airavata.helix.task.api.support.AdaptorSupport; import org.apache.airavata.monitor.platform.CountMonitor; -import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -213,7 +213,7 @@ public abstract class DataStagingTask extends AiravataTask { localDataPath = (localDataPath.endsWith(File.separator) ? localDataPath : localDataPath + File.separator) + getProcessId() + File.separator + "temp_inputs" + File.separator; try { - FileUtils.forceMkdir(new File(localDataPath)); + Files.createDirectories(new File(localDataPath).toPath()); } catch (IOException e) { throw new TaskOnFailException("Failed build directories " + localDataPath, true, e); } diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/JobSubmissionTask.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/JobSubmissionTask.java index c479901b54..4c6d787b4f 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/JobSubmissionTask.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/JobSubmissionTask.java @@ -20,6 +20,8 @@ package org.apache.airavata.helix.impl.task.submission; import java.io.File; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; import java.security.SecureRandom; import java.util.ArrayList; import java.util.List; @@ -39,7 +41,6 @@ import org.apache.airavata.helix.impl.task.submission.config.JobFactory; import org.apache.airavata.helix.impl.task.submission.config.JobManagerConfiguration; import org.apache.airavata.helix.impl.task.submission.config.RawCommandInfo; import org.apache.airavata.registry.exception.RegistryServiceException; -import org.apache.commons.io.FileUtils; import org.apache.helix.HelixManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -85,7 +86,8 @@ public abstract class JobSubmissionTask extends AiravataTask { File tempJobFile = new File( getLocalDataDir(), "job_" + Integer.toString(number) + jobManagerConfiguration.getScriptExtension()); - FileUtils.writeStringToFile(tempJobFile, scriptAsString); + Files.writeString( + tempJobFile.toPath(), scriptAsString, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); logger.info("Job submission file for process " + getProcessId() + " was created at : " + tempJobFile.getAbsolutePath()); diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapData.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapData.java index 2ba9423beb..fb9677d44b 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapData.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/GroovyMapData.java @@ -24,12 +24,11 @@ import groovy.text.GStringTemplateEngine; import groovy.text.TemplateEngine; import java.lang.reflect.Field; import java.net.URL; -import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; import java.util.Map; import org.apache.airavata.common.utils.ApplicationSettings; -import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -588,7 +587,7 @@ public class GroovyMapData { } try { - String templateStr = IOUtils.toString(templateUrl.openStream(), Charset.defaultCharset()); + String templateStr = new String(templateUrl.openStream().readAllBytes(), StandardCharsets.UTF_8); return loadFromString(templateStr); } catch (Exception e) { throw new Exception( diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/CloudJobManagerConfiguration.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/CloudJobManagerConfiguration.java index af4653ab83..a3db32cc8c 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/CloudJobManagerConfiguration.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/CloudJobManagerConfiguration.java @@ -20,11 +20,11 @@ package org.apache.airavata.helix.impl.task.submission.config.app; import java.io.File; +import java.nio.file.Paths; import org.apache.airavata.helix.impl.task.submission.config.JobManagerConfiguration; import org.apache.airavata.helix.impl.task.submission.config.OutputParser; import org.apache.airavata.helix.impl.task.submission.config.RawCommandInfo; import org.apache.airavata.helix.impl.task.submission.config.app.parser.AiravataCustomCommandOutputParser; -import org.apache.commons.io.FilenameUtils; /** * A Job Manager Configuration for executing jobs directly on a cloud VM via SSH @@ -71,7 +71,9 @@ public class CloudJobManagerConfiguration implements JobManagerConfiguration { @Override public RawCommandInfo getSubmitCommand(String workingDirectory, String filePath) { - String remoteScriptPath = workingDirectory + File.separator + FilenameUtils.getName(filePath); + String remoteScriptPath = workingDirectory + + File.separator + + Paths.get(filePath).getFileName().toString(); return new RawCommandInfo("/bin/bash " + remoteScriptPath); } diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/ForkJobConfiguration.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/ForkJobConfiguration.java index 92de27ff95..2058128cfb 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/ForkJobConfiguration.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/ForkJobConfiguration.java @@ -20,12 +20,12 @@ package org.apache.airavata.helix.impl.task.submission.config.app; import java.io.File; +import java.nio.file.Paths; import java.util.Map; import org.apache.airavata.common.model.JobManagerCommand; import org.apache.airavata.helix.impl.task.submission.config.JobManagerConfiguration; import org.apache.airavata.helix.impl.task.submission.config.OutputParser; import org.apache.airavata.helix.impl.task.submission.config.RawCommandInfo; -import org.apache.commons.io.FilenameUtils; public class ForkJobConfiguration implements JobManagerConfiguration { private final Map<JobManagerCommand, String> jobManagerCommands; @@ -87,7 +87,7 @@ public class ForkJobConfiguration implements JobManagerConfiguration { public RawCommandInfo getSubmitCommand(String workingDirectory, String forkFilePath) { return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.SUBMISSION).trim() + " " + workingDirectory + File.separator - + FilenameUtils.getName(forkFilePath)); + + Paths.get(forkFilePath).getFileName().toString()); } @Override diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/HTCondorJobConfiguration.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/HTCondorJobConfiguration.java index 810398fc03..39bf37e4f2 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/HTCondorJobConfiguration.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/HTCondorJobConfiguration.java @@ -20,12 +20,12 @@ package org.apache.airavata.helix.impl.task.submission.config.app; import java.io.File; +import java.nio.file.Paths; import java.util.Map; import org.apache.airavata.common.model.JobManagerCommand; import org.apache.airavata.helix.impl.task.submission.config.JobManagerConfiguration; import org.apache.airavata.helix.impl.task.submission.config.OutputParser; import org.apache.airavata.helix.impl.task.submission.config.RawCommandInfo; -import org.apache.commons.io.FilenameUtils; public class HTCondorJobConfiguration implements JobManagerConfiguration { private final Map<JobManagerCommand, String> jMCommands; @@ -77,7 +77,7 @@ public class HTCondorJobConfiguration implements JobManagerConfiguration { public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath) { return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.SUBMISSION).trim() + " " + workingDirectory + File.separator - + FilenameUtils.getName(pbsFilePath)); + + Paths.get(pbsFilePath).getFileName().toString()); } public String getInstalledPath() { diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/LSFJobConfiguration.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/LSFJobConfiguration.java index b4f69140a6..1884007d20 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/LSFJobConfiguration.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/LSFJobConfiguration.java @@ -20,12 +20,12 @@ package org.apache.airavata.helix.impl.task.submission.config.app; import java.io.File; +import java.nio.file.Paths; import java.util.Map; import org.apache.airavata.common.model.JobManagerCommand; import org.apache.airavata.helix.impl.task.submission.config.JobManagerConfiguration; import org.apache.airavata.helix.impl.task.submission.config.OutputParser; import org.apache.airavata.helix.impl.task.submission.config.RawCommandInfo; -import org.apache.commons.io.FilenameUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -87,7 +87,7 @@ public class LSFJobConfiguration implements JobManagerConfiguration { @Override public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath) { return new RawCommandInfo(this.installedPath + "bsub < " + workingDirectory + File.separator - + FilenameUtils.getName(pbsFilePath)); + + Paths.get(pbsFilePath).getFileName().toString()); } @Override diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/PBSJobConfiguration.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/PBSJobConfiguration.java index 9e6fa34198..c3ab50580d 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/PBSJobConfiguration.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/PBSJobConfiguration.java @@ -20,12 +20,12 @@ package org.apache.airavata.helix.impl.task.submission.config.app; import java.io.File; +import java.nio.file.Paths; import java.util.Map; import org.apache.airavata.common.model.JobManagerCommand; import org.apache.airavata.helix.impl.task.submission.config.JobManagerConfiguration; import org.apache.airavata.helix.impl.task.submission.config.OutputParser; import org.apache.airavata.helix.impl.task.submission.config.RawCommandInfo; -import org.apache.commons.io.FilenameUtils; public class PBSJobConfiguration implements JobManagerConfiguration { @@ -78,7 +78,7 @@ public class PBSJobConfiguration implements JobManagerConfiguration { public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath) { return new RawCommandInfo(this.installedPath + jobManagerCommands.get(JobManagerCommand.SUBMISSION).trim() + " " + workingDirectory + File.separator - + FilenameUtils.getName(pbsFilePath)); + + Paths.get(pbsFilePath).getFileName().toString()); } public String getInstalledPath() { diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/SlurmJobConfiguration.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/SlurmJobConfiguration.java index 15cc216f4a..3189ec7a28 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/SlurmJobConfiguration.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/SlurmJobConfiguration.java @@ -20,12 +20,12 @@ package org.apache.airavata.helix.impl.task.submission.config.app; import java.io.File; +import java.nio.file.Paths; import java.util.Map; import org.apache.airavata.common.model.JobManagerCommand; import org.apache.airavata.helix.impl.task.submission.config.JobManagerConfiguration; import org.apache.airavata.helix.impl.task.submission.config.OutputParser; import org.apache.airavata.helix.impl.task.submission.config.RawCommandInfo; -import org.apache.commons.io.FilenameUtils; public class SlurmJobConfiguration implements JobManagerConfiguration { private final Map<JobManagerCommand, String> jMCommands; @@ -77,7 +77,7 @@ public class SlurmJobConfiguration implements JobManagerConfiguration { public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath) { return new RawCommandInfo(this.installedPath + jMCommands.get(JobManagerCommand.SUBMISSION).trim() + " " + workingDirectory + File.separator - + FilenameUtils.getName(pbsFilePath)); + + Paths.get(pbsFilePath).getFileName().toString()); } public String getInstalledPath() { diff --git a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/UGEJobConfiguration.java b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/UGEJobConfiguration.java index 8660f99518..d566e8fb7c 100644 --- a/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/UGEJobConfiguration.java +++ b/airavata-api/src/main/java/org/apache/airavata/helix/impl/task/submission/config/app/UGEJobConfiguration.java @@ -20,12 +20,12 @@ package org.apache.airavata.helix.impl.task.submission.config.app; import java.io.File; +import java.nio.file.Paths; import java.util.Map; import org.apache.airavata.common.model.JobManagerCommand; import org.apache.airavata.helix.impl.task.submission.config.JobManagerConfiguration; import org.apache.airavata.helix.impl.task.submission.config.OutputParser; import org.apache.airavata.helix.impl.task.submission.config.RawCommandInfo; -import org.apache.commons.io.FilenameUtils; public class UGEJobConfiguration implements JobManagerConfiguration { private final Map<JobManagerCommand, String> jobManagerCommands; @@ -72,8 +72,8 @@ public class UGEJobConfiguration implements JobManagerConfiguration { } public RawCommandInfo getSubmitCommand(String workingDirectory, String pbsFilePath) { - return new RawCommandInfo( - this.installedPath + "qsub " + workingDirectory + File.separator + FilenameUtils.getName(pbsFilePath)); + return new RawCommandInfo(this.installedPath + "qsub " + workingDirectory + File.separator + + Paths.get(pbsFilePath).getFileName().toString()); } public String getInstalledPath() { diff --git a/airavata-api/src/main/java/org/apache/airavata/messaging/client/RabbitMQListener.java b/airavata-api/src/main/java/org/apache/airavata/messaging/client/RabbitMQListener.java index 97f32db666..e7b3b84c65 100644 --- a/airavata-api/src/main/java/org/apache/airavata/messaging/client/RabbitMQListener.java +++ b/airavata-api/src/main/java/org/apache/airavata/messaging/client/RabbitMQListener.java @@ -19,11 +19,6 @@ */ package org.apache.airavata.messaging.client; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.PosixParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,53 +32,53 @@ public class RabbitMQListener { public static void parseArguments(String[] args) { try { - Options options = new Options(); - - options.addOption("gId", true, "Gateway ID"); - options.addOption("eId", true, "Experiment ID"); - options.addOption("jId", true, "Job ID"); - options.addOption("a", false, "All Notifications"); - - CommandLineParser parser = new PosixParser(); - CommandLine cmd = parser.parse(options, args); - if (cmd.getOptions() == null || cmd.getOptions().length == 0) { + if (args == null || args.length == 0) { logger.info("You have not specified any options. We assume you need to listen to all the messages..."); gatewayId = "*"; + return; } - if (cmd.hasOption("a")) { - logger.info("Listening to all the messages..."); - gatewayId = "*"; - } else { - gatewayId = cmd.getOptionValue("gId"); - if (gatewayId == null) { + + // Simple manual parsing + java.util.Map<String, String> options = new java.util.HashMap<>(); + for (int i = 0; i < args.length; i++) { + if (args[i].equals("-a")) { + logger.info("Listening to all the messages..."); gatewayId = "*"; - logger.info( - "You have not specified a gateway id. We assume you need to listen to all the messages..."); - } - experimentId = cmd.getOptionValue("eId"); - if (experimentId == null && !gatewayId.equals("*")) { - experimentId = "*"; - logger.info( - "You have not specified a experiment id. We assume you need to listen to all the messages for the gateway with id " - + gatewayId); - } else if (experimentId == null && gatewayId.equals("*")) { - experimentId = "*"; - logger.info( - "You have not specified a experiment id and a gateway id. We assume you need to listen to all the messages..."); - } - jobId = cmd.getOptionValue("jId"); - if (jobId == null && !gatewayId.equals("*") && !experimentId.equals("*")) { - jobId = "*"; - logger.info( - "You have not specified a job id. We assume you need to listen to all the messages for the gateway with id " - + gatewayId + " with experiment id : " + experimentId); - } else if (jobId == null && gatewayId.equals("*") && experimentId.equals("*")) { - jobId = "*"; - logger.info( - "You have not specified a job Id or experiment Id or a gateway Id. We assume you need to listen to all the messages..."); + return; + } else if (args[i].equals("-gId") && i + 1 < args.length) { + options.put("gId", args[++i]); + } else if (args[i].equals("-eId") && i + 1 < args.length) { + options.put("eId", args[++i]); + } else if (args[i].equals("-jId") && i + 1 < args.length) { + options.put("jId", args[++i]); } } - } catch (ParseException e) { + + gatewayId = options.getOrDefault("gId", "*"); + if (gatewayId.equals("*")) { + logger.info("You have not specified a gateway id. We assume you need to listen to all the messages..."); + } + + experimentId = options.getOrDefault("eId", "*"); + if (experimentId.equals("*") && !gatewayId.equals("*")) { + logger.info( + "You have not specified a experiment id. We assume you need to listen to all the messages for the gateway with id " + + gatewayId); + } else if (experimentId.equals("*") && gatewayId.equals("*")) { + logger.info( + "You have not specified a experiment id and a gateway id. We assume you need to listen to all the messages..."); + } + + jobId = options.getOrDefault("jId", "*"); + if (jobId.equals("*") && !gatewayId.equals("*") && !experimentId.equals("*")) { + logger.info( + "You have not specified a job id. We assume you need to listen to all the messages for the gateway with id " + + gatewayId + " with experiment id : " + experimentId); + } else if (jobId.equals("*") && gatewayId.equals("*") && experimentId.equals("*")) { + logger.info( + "You have not specified a job Id or experiment Id or a gateway Id. We assume you need to listen to all the messages..."); + } + } catch (Exception e) { logger.error("Error while reading command line parameters", e); } } diff --git a/airavata-api/src/main/java/org/apache/airavata/metascheduler/metadata/analyzer/DataInterpreterService.java b/airavata-api/src/main/java/org/apache/airavata/metascheduler/metadata/analyzer/DataInterpreterService.java index c60d89ce1f..fbd00884dc 100644 --- a/airavata-api/src/main/java/org/apache/airavata/metascheduler/metadata/analyzer/DataInterpreterService.java +++ b/airavata-api/src/main/java/org/apache/airavata/metascheduler/metadata/analyzer/DataInterpreterService.java @@ -33,9 +33,9 @@ import org.quartz.SchedulerException; import org.quartz.SimpleScheduleBuilder; import org.quartz.Trigger; import org.quartz.TriggerBuilder; -import org.springframework.scheduling.quartz.SchedulerFactoryBean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.scheduling.quartz.SchedulerFactoryBean; import org.springframework.stereotype.Component; @Component diff --git a/airavata-api/src/main/java/org/apache/airavata/metascheduler/process/scheduling/engine/rescheduler/ProcessReschedulingService.java b/airavata-api/src/main/java/org/apache/airavata/metascheduler/process/scheduling/engine/rescheduler/ProcessReschedulingService.java index 6e504795cd..d57b16151f 100644 --- a/airavata-api/src/main/java/org/apache/airavata/metascheduler/process/scheduling/engine/rescheduler/ProcessReschedulingService.java +++ b/airavata-api/src/main/java/org/apache/airavata/metascheduler/process/scheduling/engine/rescheduler/ProcessReschedulingService.java @@ -32,9 +32,9 @@ import org.quartz.SchedulerException; import org.quartz.SimpleScheduleBuilder; import org.quartz.Trigger; import org.quartz.TriggerBuilder; -import org.springframework.scheduling.quartz.SchedulerFactoryBean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.scheduling.quartz.SchedulerFactoryBean; import org.springframework.stereotype.Component; /** diff --git a/airavata-api/src/main/java/org/apache/airavata/monitor/cluster/ClusterStatusMonitorJobScheduler.java b/airavata-api/src/main/java/org/apache/airavata/monitor/cluster/ClusterStatusMonitorJobScheduler.java index 4d9e3b810a..323dd6e0fa 100644 --- a/airavata-api/src/main/java/org/apache/airavata/monitor/cluster/ClusterStatusMonitorJobScheduler.java +++ b/airavata-api/src/main/java/org/apache/airavata/monitor/cluster/ClusterStatusMonitorJobScheduler.java @@ -28,9 +28,9 @@ import org.quartz.JobDetail; import org.quartz.Scheduler; import org.quartz.SchedulerException; import org.quartz.Trigger; -import org.springframework.scheduling.quartz.SchedulerFactoryBean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.scheduling.quartz.SchedulerFactoryBean; public class ClusterStatusMonitorJobScheduler { private static final Logger logger = LoggerFactory.getLogger(ClusterStatusMonitorJobScheduler.class); diff --git a/airavata-api/src/main/java/org/apache/airavata/monitor/compute/ComputationalResourceMonitoringService.java b/airavata-api/src/main/java/org/apache/airavata/monitor/compute/ComputationalResourceMonitoringService.java index d153ae1124..75da6ca246 100644 --- a/airavata-api/src/main/java/org/apache/airavata/monitor/compute/ComputationalResourceMonitoringService.java +++ b/airavata-api/src/main/java/org/apache/airavata/monitor/compute/ComputationalResourceMonitoringService.java @@ -33,9 +33,9 @@ import org.quartz.SchedulerException; import org.quartz.SimpleScheduleBuilder; import org.quartz.Trigger; import org.quartz.TriggerBuilder; -import org.springframework.scheduling.quartz.SchedulerFactoryBean; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.scheduling.quartz.SchedulerFactoryBean; /** * Computational Resource Monitoring Service diff --git a/airavata-api/src/main/java/org/apache/airavata/orchestrator/impl/SimpleOrchestratorImpl.java b/airavata-api/src/main/java/org/apache/airavata/orchestrator/impl/SimpleOrchestratorImpl.java index e5468f004c..28c0c068ec 100644 --- a/airavata-api/src/main/java/org/apache/airavata/orchestrator/impl/SimpleOrchestratorImpl.java +++ b/airavata-api/src/main/java/org/apache/airavata/orchestrator/impl/SimpleOrchestratorImpl.java @@ -19,6 +19,7 @@ */ package org.apache.airavata.orchestrator.impl; +import jakarta.annotation.PostConstruct; import java.io.File; import java.net.URI; import java.net.URISyntaxException; @@ -57,13 +58,13 @@ import org.apache.airavata.common.model.TaskStatus; import org.apache.airavata.common.model.TaskTypes; import org.apache.airavata.common.utils.AiravataUtils; import org.apache.airavata.config.AiravataServerProperties; -import org.apache.airavata.model.util.ExperimentModelUtil; import org.apache.airavata.orchestrator.exception.OrchestratorException; import org.apache.airavata.orchestrator.job.GFACPassiveJobSubmitter; import org.apache.airavata.orchestrator.job.JobSubmitter; import org.apache.airavata.orchestrator.utils.OrchestratorUtils; import org.apache.airavata.registry.exception.RegistryServiceException; import org.apache.airavata.service.registry.RegistryService; +import org.apache.airavata.util.ExperimentModelUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -112,7 +113,7 @@ public class SimpleOrchestratorImpl extends AbstractOrchestrator { } } - @jakarta.annotation.PostConstruct + @PostConstruct public void init() throws OrchestratorException { logger.info("[BEAN-INIT] SimpleOrchestratorImpl.init() called"); initialize(properties); diff --git a/airavata-api/src/main/java/org/apache/airavata/registry/utils/migration/MappingToolRunner.java b/airavata-api/src/main/java/org/apache/airavata/registry/utils/migration/MappingToolRunner.java index 641c6ffc61..1b3fc56228 100644 --- a/airavata-api/src/main/java/org/apache/airavata/registry/utils/migration/MappingToolRunner.java +++ b/airavata-api/src/main/java/org/apache/airavata/registry/utils/migration/MappingToolRunner.java @@ -19,15 +19,15 @@ */ package org.apache.airavata.registry.utils.migration; +import jakarta.persistence.EntityManagerFactory; import org.apache.airavata.common.utils.DBInitConfig; import org.apache.airavata.common.utils.JPAUtils; -import jakarta.persistence.EntityManagerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Utility class for generating database schema DDL scripts using Hibernate. - * + * * Note: Consider migrating to a proper database migration tool like Flyway or Liquibase * for production schema management. */ @@ -44,7 +44,7 @@ public class MappingToolRunner { /** * Generate database schema DDL script using Hibernate SchemaExport. - * + * * @param dbInitConfig Database configuration * @param outputFile Output file path for the DDL script * @param persistenceUnitName Name of the persistence unit @@ -63,25 +63,25 @@ public class MappingToolRunner { dbInitConfig.getUser(), dbInitConfig.getPassword(), dbInitConfig.getValidationQuery()); - + // Note: Hibernate 6 schema export API has changed significantly from Hibernate 5. // For now, this method logs a warning. To implement full schema export: // 1. Use Hibernate's SchemaManagementTool with proper SourceDescriptor and TargetDescriptor // 2. Or use a database migration tool like Flyway/Liquibase // 3. Or use hibernate.hbm2ddl.auto=update in development // 4. Or use Hibernate's SchemaExport programmatically with proper Hibernate 6 API - + logger.warn("Schema export via MappingToolRunner is not fully implemented for Hibernate 6."); logger.warn("Consider using hibernate.hbm2ddl.auto=update or a migration tool like Flyway/Liquibase."); logger.warn("Requested output file: {}", outputFile); - + // TODO: Implement proper Hibernate 6 schema export API // The API requires: // - SourceDescriptor for source metadata // - TargetDescriptor for output target // - Proper ExecutionOptions // See: org.hibernate.tool.schema.spi.SchemaManagementTool - + } catch (Exception ex) { logger.error("Failed to generate schema DDL script", ex); throw new RuntimeException("Failed to generate schema DDL script using Hibernate", ex); diff --git a/airavata-api/src/main/java/org/apache/airavata/security/KeyCloakSecurityManager.java b/airavata-api/src/main/java/org/apache/airavata/security/KeyCloakSecurityManager.java index b0b263568f..a250432b6c 100644 --- a/airavata-api/src/main/java/org/apache/airavata/security/KeyCloakSecurityManager.java +++ b/airavata-api/src/main/java/org/apache/airavata/security/KeyCloakSecurityManager.java @@ -19,6 +19,8 @@ */ package org.apache.airavata.security; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; @@ -55,7 +57,6 @@ import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.message.BasicNameValuePair; import org.apache.http.util.EntityUtils; -import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -112,6 +113,7 @@ public class KeyCloakSecurityManager implements AiravataSecurityManager { private final AiravataServerProperties properties; private final AuthzCacheManagerFactory authzCacheManagerFactory; private final GatewayGroupsInitializer gatewayGroupsInitializer; + private final ObjectMapper objectMapper = new ObjectMapper(); public KeyCloakSecurityManager( RegistryService registryService, @@ -245,9 +247,9 @@ public class KeyCloakSecurityManager implements AiravataSecurityManager { initServiceClients(); Gateway gateway = registryService.getGateway(gatewayId); String tokenURL = getTokenEndpoint(gatewayId); - JSONObject clientCredentials = + JsonNode clientCredentials = getClientCredentials(tokenURL, gateway.getOauthClientId(), gateway.getOauthClientSecret()); - String accessToken = clientCredentials.getString("access_token"); + String accessToken = clientCredentials.get("access_token").asText(); AuthzToken authzToken = new AuthzToken(accessToken); authzToken.putToClaimsMap(Constants.GATEWAY_ID, gatewayId); authzToken.putToClaimsMap(Constants.USER_NAME, gateway.getOauthClientId()); @@ -277,16 +279,16 @@ public class KeyCloakSecurityManager implements AiravataSecurityManager { GatewayResourceProfile gwrp = registryService.getGatewayResourceProfile(gatewayId); String identityServerRealm = gwrp.getIdentityServerTenant(); String openIdConnectUrl = getOpenIDConfigurationUrl(identityServerRealm); - JSONObject openIdConnectConfig = new JSONObject(getFromUrl(openIdConnectUrl, null)); - String userInfoEndPoint = openIdConnectConfig.getString("userinfo_endpoint"); - JSONObject userInfo = new JSONObject(getFromUrl(userInfoEndPoint, token)); + JsonNode openIdConnectConfig = objectMapper.readTree(getFromUrl(openIdConnectUrl, null)); + String userInfoEndPoint = openIdConnectConfig.get("userinfo_endpoint").asText(); + JsonNode userInfo = objectMapper.readTree(getFromUrl(userInfoEndPoint, token)); return new UserInfo() - .setSub(userInfo.getString("sub")) - .setFullName(userInfo.getString("name")) - .setFirstName(userInfo.getString("given_name")) - .setLastName(userInfo.getString("family_name")) - .setEmailAddress(userInfo.getString("email")) - .setUsername(userInfo.getString("preferred_username")); + .setSub(userInfo.get("sub").asText()) + .setFullName(userInfo.get("name").asText()) + .setFirstName(userInfo.get("given_name").asText()) + .setLastName(userInfo.get("family_name").asText()) + .setEmailAddress(userInfo.get("email").asText()) + .setUsername(userInfo.get("preferred_username").asText()); } private GatewayGroupMembership getGatewayGroupMembership(String username, String token, String gatewayId) @@ -344,11 +346,11 @@ public class KeyCloakSecurityManager implements AiravataSecurityManager { private String getTokenEndpoint(String gatewayId) throws Exception { String openIdConnectUrl = getOpenIDConfigurationUrl(gatewayId); - JSONObject openIdConnectConfig = new JSONObject(getFromUrl(openIdConnectUrl, null)); - return openIdConnectConfig.getString("token_endpoint"); + JsonNode openIdConnectConfig = objectMapper.readTree(getFromUrl(openIdConnectUrl, null)); + return openIdConnectConfig.get("token_endpoint").asText(); } - public JSONObject getClientCredentials(String tokenURL, String clientId, String clientSecret) throws IOException { + public JsonNode getClientCredentials(String tokenURL, String clientId, String clientSecret) throws IOException { CloseableHttpClient httpClient = HttpClients.createSystem(); @@ -362,7 +364,7 @@ public class KeyCloakSecurityManager implements AiravataSecurityManager { httpPost.setEntity(entity); try (CloseableHttpResponse response = httpClient.execute(httpPost)) { String responseBody = EntityUtils.toString(response.getEntity()); - return new JSONObject(responseBody); + return objectMapper.readTree(responseBody); } catch (IOException e) { throw new RuntimeException(e); } finally { diff --git a/airavata-api/src/main/java/org/apache/airavata/security/interceptor/SecurityModule.java b/airavata-api/src/main/java/org/apache/airavata/security/interceptor/SecurityModule.java index 2e143b7b39..560466b00d 100644 --- a/airavata-api/src/main/java/org/apache/airavata/security/interceptor/SecurityModule.java +++ b/airavata-api/src/main/java/org/apache/airavata/security/interceptor/SecurityModule.java @@ -11,7 +11,8 @@ * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, -* software distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. diff --git a/airavata-api/src/main/java/org/apache/airavata/security/userstore/JDBCUserStore.java b/airavata-api/src/main/java/org/apache/airavata/security/userstore/JDBCUserStore.java index 063775e5e4..0312f3f1d2 100644 --- a/airavata-api/src/main/java/org/apache/airavata/security/userstore/JDBCUserStore.java +++ b/airavata-api/src/main/java/org/apache/airavata/security/userstore/JDBCUserStore.java @@ -11,7 +11,8 @@ * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, -* software distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. @@ -59,9 +60,9 @@ public class JDBCUserStore extends AbstractJDBCUserStore { public boolean authenticate(String userName, Object credentials) throws UserStoreException { try { String password = passwordDigester.getPasswordHashValue((String) credentials); - UsernamePasswordAuthenticationToken authRequest = - new UsernamePasswordAuthenticationToken(userName, password); - + UsernamePasswordAuthenticationToken authRequest = + new UsernamePasswordAuthenticationToken(userName, password); + Authentication authentication = authenticationProvider.authenticate(authRequest); return authentication != null && authentication.isAuthenticated(); } catch (BadCredentialsException e) { @@ -155,12 +156,13 @@ public class JDBCUserStore extends AbstractJDBCUserStore { protected void initializeDatabaseLookup(String passwordColumn, String userTable, String userNameColumn) throws ApplicationSettingsException, UserStoreException { try { - DBUtil dbUtil = new DBUtil(getDatabaseURL(), getDatabaseUserName(), getDatabasePassword(), getDatabaseDriver()); + DBUtil dbUtil = + new DBUtil(getDatabaseURL(), getDatabaseUserName(), getDatabasePassword(), getDatabaseDriver()); dataSource = dbUtil.getDataSource(); // Create user details service - UserDetailsService userDetailsService = new JdbcUserDetailsService( - dataSource, userTable, userNameColumn, passwordColumn); + UserDetailsService userDetailsService = + new JdbcUserDetailsService(dataSource, userTable, userNameColumn, passwordColumn); // Create password encoder adapter PasswordEncoder passwordEncoder = new PasswordDigesterEncoder(passwordDigester); @@ -188,8 +190,8 @@ public class JDBCUserStore extends AbstractJDBCUserStore { private final String userNameColumn; private final String passwordColumn; - public JdbcUserDetailsService(DataSource dataSource, String userTable, - String userNameColumn, String passwordColumn) { + public JdbcUserDetailsService( + DataSource dataSource, String userTable, String userNameColumn, String passwordColumn) { this.jdbcTemplate = new JdbcTemplate(dataSource); this.userTable = userTable; this.userNameColumn = userNameColumn; @@ -198,18 +200,17 @@ public class JDBCUserStore extends AbstractJDBCUserStore { @Override public UserDetails loadUserByUsername(String username) throws UsernameNotFoundException { - String sql = String.format("SELECT %s FROM %s WHERE %s = ?", - passwordColumn, userTable, userNameColumn); - + String sql = String.format("SELECT %s FROM %s WHERE %s = ?", passwordColumn, userTable, userNameColumn); + try { String password = jdbcTemplate.queryForObject(sql, String.class, username); if (password == null) { throw new UsernameNotFoundException("User not found: " + username); } return User.withUsername(username) - .password(password) - .authorities("ROLE_USER") - .build(); + .password(password) + .authorities("ROLE_USER") + .build(); } catch (org.springframework.dao.EmptyResultDataAccessException e) { throw new UsernameNotFoundException("User not found: " + username, e); } diff --git a/airavata-api/src/main/java/org/apache/airavata/security/userstore/LDAPUserStore.java b/airavata-api/src/main/java/org/apache/airavata/security/userstore/LDAPUserStore.java index f948335351..6b0abeafc1 100644 --- a/airavata-api/src/main/java/org/apache/airavata/security/userstore/LDAPUserStore.java +++ b/airavata-api/src/main/java/org/apache/airavata/security/userstore/LDAPUserStore.java @@ -11,7 +11,8 @@ * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, -* software distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +* software distributed under the License is distributed on an +* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. @@ -51,9 +52,9 @@ public class LDAPUserStore implements UserStore { public boolean authenticate(String userName, Object credentials) throws UserStoreException { try { String password = passwordDigester.getPasswordHashValue((String) credentials); - UsernamePasswordAuthenticationToken authRequest = - new UsernamePasswordAuthenticationToken(userName, password); - + UsernamePasswordAuthenticationToken authRequest = + new UsernamePasswordAuthenticationToken(userName, password); + Authentication authentication = ldapAuthenticationProvider.authenticate(authRequest); return authentication != null && authentication.isAuthenticated(); } catch (BadCredentialsException e) { @@ -126,8 +127,8 @@ public class LDAPUserStore implements UserStore { initializeLDAP(url, systemUser, systemUserPassword, userTemplate); } - protected void initializeLDAP( - String ldapUrl, String systemUser, String systemUserPassword, String userNameTemplate) throws UserStoreException { + protected void initializeLDAP(String ldapUrl, String systemUser, String systemUserPassword, String userNameTemplate) + throws UserStoreException { try { // Create LDAP context source @@ -154,8 +155,7 @@ public class LDAPUserStore implements UserStore { } } - FilterBasedLdapUserSearch userSearch = new FilterBasedLdapUserSearch( - searchBase, searchFilter, contextSource); + FilterBasedLdapUserSearch userSearch = new FilterBasedLdapUserSearch(searchBase, searchFilter, contextSource); // Create authenticator PasswordComparisonAuthenticator authenticator = new PasswordComparisonAuthenticator(contextSource); @@ -163,8 +163,7 @@ public class LDAPUserStore implements UserStore { authenticator.setPasswordEncoder(new PasswordDigesterEncoder(passwordDigester)); // Create authorities populator (empty for now - can be extended) - DefaultLdapAuthoritiesPopulator authoritiesPopulator = - new DefaultLdapAuthoritiesPopulator(contextSource, ""); + DefaultLdapAuthoritiesPopulator authoritiesPopulator = new DefaultLdapAuthoritiesPopulator(contextSource, ""); // Create authentication provider ldapAuthenticationProvider = new LdapAuthenticationProvider(authenticator, authoritiesPopulator); @@ -173,7 +172,8 @@ public class LDAPUserStore implements UserStore { /** * Password encoder adapter for PasswordDigester */ - private static class PasswordDigesterEncoder implements org.springframework.security.crypto.password.PasswordEncoder { + private static class PasswordDigesterEncoder + implements org.springframework.security.crypto.password.PasswordEncoder { private final PasswordDigester passwordDigester; public PasswordDigesterEncoder(PasswordDigester passwordDigester) { diff --git a/airavata-api/src/main/java/org/apache/airavata/service/orchestrator/OrchestratorService.java b/airavata-api/src/main/java/org/apache/airavata/service/orchestrator/OrchestratorService.java index 19f51674e6..6c52a8aace 100644 --- a/airavata-api/src/main/java/org/apache/airavata/service/orchestrator/OrchestratorService.java +++ b/airavata-api/src/main/java/org/apache/airavata/service/orchestrator/OrchestratorService.java @@ -74,14 +74,13 @@ import org.apache.airavata.messaging.core.Publisher; import org.apache.airavata.messaging.core.Subscriber; import org.apache.airavata.messaging.core.Type; import org.apache.airavata.metascheduler.core.api.ProcessScheduler; -import org.apache.airavata.model.util.ExperimentModelUtil; import org.apache.airavata.orchestrator.exception.OrchestratorException; import org.apache.airavata.orchestrator.impl.SimpleOrchestratorImpl; import org.apache.airavata.orchestrator.schedule.HostScheduler; import org.apache.airavata.orchestrator.utils.OrchestratorConstants; import org.apache.airavata.registry.exception.RegistryServiceException; import org.apache.airavata.service.registry.RegistryService; -import org.apache.commons.lang3.StringUtils; +import org.apache.airavata.util.ExperimentModelUtil; import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; @@ -288,7 +287,7 @@ public class OrchestratorService { filePathList.add(uri); } } - pi.setValue(StringUtils.join(filePathList, ',')); + pi.setValue(String.join(",", filePathList)); } } } diff --git a/airavata-api/src/main/java/org/apache/airavata/service/security/CredentialStoreService.java b/airavata-api/src/main/java/org/apache/airavata/service/security/CredentialStoreService.java index de25c9b136..b15dfe1ee1 100644 --- a/airavata-api/src/main/java/org/apache/airavata/service/security/CredentialStoreService.java +++ b/airavata-api/src/main/java/org/apache/airavata/service/security/CredentialStoreService.java @@ -23,6 +23,7 @@ import java.io.ByteArrayInputStream; import java.security.cert.CertificateException; import java.security.cert.CertificateFactory; import java.security.cert.X509Certificate; +import java.util.Base64; import java.util.List; import java.util.UUID; import java.util.stream.Collectors; @@ -43,7 +44,6 @@ import org.apache.airavata.credential.model.SummaryType; import org.apache.airavata.credential.utils.CredentialStoreDBInitConfig; import org.apache.airavata.credential.utils.TokenGenerator; import org.apache.airavata.credential.utils.Utility; -import org.apache.commons.codec.binary.Base64; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; @@ -159,8 +159,8 @@ public class CredentialStoreService { String token = TokenGenerator.generateToken( certificateCredential.getCommunityUser().getGatewayName(), null); credential.setToken(token); - Base64 encoder = new Base64(64); - byte[] decoded = encoder.decode(certificateCredential + Base64.Decoder decoder = Base64.getMimeDecoder(); + byte[] decoded = decoder.decode(certificateCredential .getX509Cert() .replaceAll("-----BEGIN CERTIFICATE-----", "") .replaceAll("-----END CERTIFICATE-----", "")); diff --git a/airavata-api/src/main/java/org/apache/airavata/model/util/AppDeploymentUtil.java b/airavata-api/src/main/java/org/apache/airavata/util/AppDeploymentUtil.java similarity index 97% rename from airavata-api/src/main/java/org/apache/airavata/model/util/AppDeploymentUtil.java rename to airavata-api/src/main/java/org/apache/airavata/util/AppDeploymentUtil.java index bafad3796a..0d6ee42a77 100644 --- a/airavata-api/src/main/java/org/apache/airavata/model/util/AppDeploymentUtil.java +++ b/airavata-api/src/main/java/org/apache/airavata/util/AppDeploymentUtil.java @@ -17,7 +17,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.airavata.model.util; +package org.apache.airavata.util; import org.apache.airavata.common.model.ApplicationDeploymentDescription; import org.apache.airavata.common.model.SetEnvPaths; diff --git a/airavata-api/src/main/java/org/apache/airavata/model/util/AppInterfaceUtil.java b/airavata-api/src/main/java/org/apache/airavata/util/AppInterfaceUtil.java similarity index 98% rename from airavata-api/src/main/java/org/apache/airavata/model/util/AppInterfaceUtil.java rename to airavata-api/src/main/java/org/apache/airavata/util/AppInterfaceUtil.java index 09b90a0f10..bc88b5b081 100644 --- a/airavata-api/src/main/java/org/apache/airavata/model/util/AppInterfaceUtil.java +++ b/airavata-api/src/main/java/org/apache/airavata/util/AppInterfaceUtil.java @@ -17,7 +17,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.airavata.model.util; +package org.apache.airavata.util; import java.util.List; import org.apache.airavata.common.model.ApplicationInterfaceDescription; diff --git a/airavata-api/src/main/java/org/apache/airavata/model/util/ExecutionType.java b/airavata-api/src/main/java/org/apache/airavata/util/ExecutionType.java similarity index 95% rename from airavata-api/src/main/java/org/apache/airavata/model/util/ExecutionType.java rename to airavata-api/src/main/java/org/apache/airavata/util/ExecutionType.java index a5e63fec50..88ca942892 100644 --- a/airavata-api/src/main/java/org/apache/airavata/model/util/ExecutionType.java +++ b/airavata-api/src/main/java/org/apache/airavata/util/ExecutionType.java @@ -17,7 +17,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.airavata.model.util; +package org.apache.airavata.util; public enum ExecutionType { UNKNOWN, diff --git a/airavata-api/src/main/java/org/apache/airavata/model/util/ExperimentModelUtil.java b/airavata-api/src/main/java/org/apache/airavata/util/ExperimentModelUtil.java similarity index 99% rename from airavata-api/src/main/java/org/apache/airavata/model/util/ExperimentModelUtil.java rename to airavata-api/src/main/java/org/apache/airavata/util/ExperimentModelUtil.java index d32cbaac9d..922a59c7b5 100644 --- a/airavata-api/src/main/java/org/apache/airavata/model/util/ExperimentModelUtil.java +++ b/airavata-api/src/main/java/org/apache/airavata/util/ExperimentModelUtil.java @@ -17,7 +17,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.airavata.model.util; +package org.apache.airavata.util; import java.util.List; import org.apache.airavata.common.model.ComputationalResourceSchedulingModel; diff --git a/airavata-api/src/main/java/org/apache/airavata/model/util/GroupComputeResourcePreferenceUtil.java b/airavata-api/src/main/java/org/apache/airavata/util/GroupComputeResourcePreferenceUtil.java similarity index 98% rename from airavata-api/src/main/java/org/apache/airavata/model/util/GroupComputeResourcePreferenceUtil.java rename to airavata-api/src/main/java/org/apache/airavata/util/GroupComputeResourcePreferenceUtil.java index 92c5fe41a7..77c5d40e2d 100644 --- a/airavata-api/src/main/java/org/apache/airavata/model/util/GroupComputeResourcePreferenceUtil.java +++ b/airavata-api/src/main/java/org/apache/airavata/util/GroupComputeResourcePreferenceUtil.java @@ -17,7 +17,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.airavata.model.util; +package org.apache.airavata.util; import org.apache.airavata.common.model.ComputeResourceReservation; import org.apache.airavata.common.model.ComputeResourceType; diff --git a/airavata-api/src/main/java/org/apache/airavata/model/util/ProjectModelUtil.java b/airavata-api/src/main/java/org/apache/airavata/util/ProjectModelUtil.java similarity index 96% rename from airavata-api/src/main/java/org/apache/airavata/model/util/ProjectModelUtil.java rename to airavata-api/src/main/java/org/apache/airavata/util/ProjectModelUtil.java index 6a6e85e170..367392a86b 100644 --- a/airavata-api/src/main/java/org/apache/airavata/model/util/ProjectModelUtil.java +++ b/airavata-api/src/main/java/org/apache/airavata/util/ProjectModelUtil.java @@ -17,7 +17,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.airavata.model.util; +package org.apache.airavata.util; import org.apache.airavata.common.model.Project; diff --git a/airavata-api/src/test/java/org/apache/airavata/api/thrift/client/TestSSLClient.java b/airavata-api/src/test/java/org/apache/airavata/api/thrift/client/TestSSLClient.java index 0dcfc80845..55d59830d6 100644 --- a/airavata-api/src/test/java/org/apache/airavata/api/thrift/client/TestSSLClient.java +++ b/airavata-api/src/test/java/org/apache/airavata/api/thrift/client/TestSSLClient.java @@ -23,11 +23,11 @@ import java.io.File; import java.io.FileInputStream; import java.security.KeyStore; import java.security.cert.X509Certificate; +import java.util.Base64; import org.apache.airavata.credential.model.CertificateCredential; import org.apache.airavata.credential.model.CommunityUser; import org.apache.airavata.credential.model.SSHCredential; import org.apache.airavata.service.security.CredentialStoreService; -import org.apache.commons.codec.binary.Base64; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -81,11 +81,11 @@ public class TestSSLClient { char[] password = "airavata".toCharArray(); ks.load(fis, password); x509Certificates[0] = (X509Certificate) ks.getCertificate("airavata"); - Base64 encoder = new Base64(64); + Base64.Encoder encoder = Base64.getMimeEncoder(64, "\n".getBytes()); String cert_begin = "-----BEGIN CERTIFICATE-----\n"; String end_cert = "-----END CERTIFICATE-----"; byte[] derCert = x509Certificates[0].getEncoded(); - String pemCertPre = new String(encoder.encode(derCert)); + String pemCertPre = encoder.encodeToString(derCert); String pemCert = cert_begin + pemCertPre + end_cert; certificateCredential.setX509Cert(pemCert); String token = credentialService.addCertificateCredential(certificateCredential); diff --git a/airavata-api/src/test/java/org/apache/airavata/api/thrift/server/TestOrchestratorServiceServer.java b/airavata-api/src/test/java/org/apache/airavata/api/thrift/server/TestOrchestratorServiceServer.java index 8e0f31da4c..ab1e23cbe1 100644 --- a/airavata-api/src/test/java/org/apache/airavata/api/thrift/server/TestOrchestratorServiceServer.java +++ b/airavata-api/src/test/java/org/apache/airavata/api/thrift/server/TestOrchestratorServiceServer.java @@ -28,11 +28,11 @@ import org.apache.airavata.common.model.InputDataObjectType; import org.apache.airavata.common.model.OutputDataObjectType; import org.apache.airavata.common.model.UserConfigurationDataModel; import org.apache.airavata.config.AiravataServerProperties; -import org.apache.airavata.model.util.ExperimentModelUtil; import org.apache.airavata.orchestrator.exception.OrchestratorException; import org.apache.airavata.registry.exception.RegistryServiceException; import org.apache.airavata.service.orchestrator.OrchestratorService; import org.apache.airavata.service.registry.RegistryService; +import org.apache.airavata.util.ExperimentModelUtil; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.slf4j.Logger; diff --git a/airavata-api/src/test/java/org/apache/airavata/api/thrift/util/ThriftClientPoolTest.java b/airavata-api/src/test/java/org/apache/airavata/api/thrift/util/ThriftClientPoolTest.java index 8700d21e45..6e3e8d2aa5 100644 --- a/airavata-api/src/test/java/org/apache/airavata/api/thrift/util/ThriftClientPoolTest.java +++ b/airavata-api/src/test/java/org/apache/airavata/api/thrift/util/ThriftClientPoolTest.java @@ -19,138 +19,57 @@ */ package org.apache.airavata.api.thrift.util; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import java.io.PrintWriter; -import java.io.StringWriter; import org.apache.airavata.api.model.BaseAPI; import org.apache.airavata.common.exception.ApplicationSettingsException; -import org.apache.airavata.common.utils.ApplicationSettings; -import org.apache.commons.pool2.impl.AbandonedConfig; -import org.apache.commons.pool2.impl.GenericObjectPoolConfig; import org.apache.thrift.TException; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.TTransport; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Import; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.bean.override.mockito.MockitoBean; -@SpringBootTest( - classes = {org.apache.airavata.config.JpaConfig.class, ThriftClientPoolTest.TestConfiguration.class}, - properties = { - "spring.main.allow-bean-definition-overriding=true", - "spring.autoconfigure.exclude=org.springframework.boot.autoconfigure.data.jpa.JpaRepositoriesAutoConfiguration" - }) -@TestPropertySource(locations = "classpath:airavata.properties") public class ThriftClientPoolTest { - @MockitoBean private BaseAPI.Client mockClient; - - public ThriftClientPoolTest(BaseAPI.Client mockClient) { - this.mockClient = mockClient; + private TProtocol mockInputProtocol; + private TProtocol mockOutputProtocol; + private TTransport mockInputTransport; + private TTransport mockOutputTransport; + + @BeforeEach + public void setUp() { + mockClient = mock(BaseAPI.Client.class); + mockInputProtocol = mock(TProtocol.class); + mockOutputProtocol = mock(TProtocol.class); + mockInputTransport = mock(TTransport.class); + mockOutputTransport = mock(TTransport.class); + + when(mockClient.getInputProtocol()).thenReturn(mockInputProtocol); + when(mockClient.getOutputProtocol()).thenReturn(mockOutputProtocol); + when(mockInputProtocol.getTransport()).thenReturn(mockInputTransport); + when(mockOutputProtocol.getTransport()).thenReturn(mockOutputTransport); + when(mockInputTransport.isOpen()).thenReturn(true); + when(mockOutputTransport.isOpen()).thenReturn(true); } @Test public void testWithDefaultConfig() throws TException { when(mockClient.getAPIVersion()).thenReturn("0.19"); - when(mockClient.getInputProtocol().getTransport().isOpen()).thenReturn(true); - when(mockClient.getOutputProtocol().getTransport().isOpen()).thenReturn(true); - GenericObjectPoolConfig<BaseAPI.Client> poolConfig = new GenericObjectPoolConfig<>(); - ThriftClientPool<BaseAPI.Client> thriftClientPool = - new ThriftClientPool<>((protocol) -> mockClient, () -> null, poolConfig); - BaseAPI.Client client = thriftClientPool.getResource(); + var poolConfig = new ThriftClientPool.PoolConfig(); + var mockProtocol = mock(TProtocol.class); + var thriftClientPool = new ThriftClientPool<>((protocol) -> mockClient, () -> mockProtocol, poolConfig); + var client = thriftClientPool.getResource(); thriftClientPool.returnResource(client); thriftClientPool.close(); - verify(mockClient.getInputProtocol().getTransport()).close(); - verify(mockClient.getOutputProtocol().getTransport()).close(); - } - - @Test - public void testWithAbandonConfigAndAbandoned() throws TException { - - when(mockClient.getAPIVersion()).thenReturn("0.19"); - when(mockClient.getInputProtocol().getTransport().isOpen()).thenReturn(true); - when(mockClient.getOutputProtocol().getTransport().isOpen()).thenReturn(true); - - GenericObjectPoolConfig<BaseAPI.Client> poolConfig = new GenericObjectPoolConfig<>(); - // timeBetweenEvictionRunsMillis must be positive for abandoned removal on - // maintenance to run - poolConfig.setTimeBetweenEvictionRunsMillis(1); - AbandonedConfig abandonedConfig = new AbandonedConfig(); - abandonedConfig.setRemoveAbandonedTimeout(1); - abandonedConfig.setRemoveAbandonedOnMaintenance(true); - abandonedConfig.setLogAbandoned(true); - StringWriter log = new StringWriter(); - assertEquals(0, log.toString().length(), "Initial length of log is 0"); - PrintWriter logWriter = new PrintWriter(log); - abandonedConfig.setLogWriter(logWriter); - ThriftClientPool<BaseAPI.Client> thriftClientPool = - new ThriftClientPool<>((protocol) -> mockClient, () -> null, poolConfig, abandonedConfig); - thriftClientPool.getResource(); - try { - // Sleep long enough for the client to be considered abandoned - Thread.sleep(1001); - thriftClientPool.close(); - } catch (InterruptedException e) { - fail("sleep interrupted"); - } - - assertTrue(log.toString().length() > 0); - // The stack trace should contain this method's name - assertTrue(log.toString().contains("testWithAbandonConfigAndAbandoned")); - - // Verify client is destroyed when abandoned - verify(mockClient.getInputProtocol().getTransport(), times(1)).close(); - verify(mockClient.getOutputProtocol().getTransport(), times(1)).close(); - } - - @Test - public void testWithAbandonConfigAndAbandonedAndNotLogged() throws TException { - - when(mockClient.getAPIVersion()).thenReturn("0.19"); - when(mockClient.getInputProtocol().getTransport().isOpen()).thenReturn(true); - when(mockClient.getOutputProtocol().getTransport().isOpen()).thenReturn(true); - - GenericObjectPoolConfig<BaseAPI.Client> poolConfig = new GenericObjectPoolConfig<>(); - // timeBetweenEvictionRunsMillis must be positive for abandoned removal on - // maintenance to run - poolConfig.setTimeBetweenEvictionRunsMillis(1); - AbandonedConfig abandonedConfig = new AbandonedConfig(); - abandonedConfig.setRemoveAbandonedTimeout(1); - abandonedConfig.setRemoveAbandonedOnMaintenance(true); - abandonedConfig.setLogAbandoned(false); - // Setup log writer so we can verify that nothing was logged - StringWriter log = new StringWriter(); - assertEquals(0, log.toString().length(), "Initial length of log is 0"); - PrintWriter logWriter = new PrintWriter(log); - abandonedConfig.setLogWriter(logWriter); - ThriftClientPool<BaseAPI.Client> thriftClientPool = - new ThriftClientPool<>((protocol) -> mockClient, () -> null, poolConfig, abandonedConfig); - thriftClientPool.getResource(); - try { - // Sleep long enough for the client to be considered abandoned - Thread.sleep(1001); - thriftClientPool.close(); - } catch (InterruptedException e) { - fail("sleep interrupted"); - } - - // Verify that nothing was logged - assertEquals(0, log.toString().length()); - - // Verify client is destroyed when abandoned - verify(mockClient.getInputProtocol().getTransport(), times(1)).close(); - verify(mockClient.getOutputProtocol().getTransport(), times(1)).close(); + verify(mockInputTransport).close(); + verify(mockOutputTransport).close(); } /** @@ -163,44 +82,17 @@ public class ThriftClientPoolTest { @Test @Disabled("Test requires long wait time to account for default removeAbandonedTimeout") public void testWithDefaultAbandonedRemovalEnabled() throws TException, ApplicationSettingsException { - when(mockClient.getAPIVersion()).thenReturn("0.19"); - when(mockClient.getInputProtocol().getTransport().isOpen()).thenReturn(true); - when(mockClient.getOutputProtocol().getTransport().isOpen()).thenReturn(true); - GenericObjectPoolConfig<BaseAPI.Client> poolConfig = new GenericObjectPoolConfig<>(); - // timeBetweenEvictionRunsMillis must be positive for abandoned removal on - // maintenance to run + var poolConfig = new ThriftClientPool.PoolConfig(); poolConfig.setTimeBetweenEvictionRunsMillis(1); - ApplicationSettings.setSetting("airavata.thrift-client-pool-abandoned-removal-enabled", "true"); - ThriftClientPool<BaseAPI.Client> thriftClientPool = - new ThriftClientPool<>((protocol) -> mockClient, () -> null, poolConfig); + var mockProtocol = mock(TProtocol.class); + var thriftClientPool = new ThriftClientPool<>((protocol) -> mockClient, () -> mockProtocol, poolConfig); thriftClientPool.getResource(); - try { - // Sleep long enough for the client to be considered abandoned - // Default removeAbandonedTimeout is 300 seconds - Thread.sleep(new AbandonedConfig().getRemoveAbandonedTimeout() * 1000 + 1); - thriftClientPool.close(); - } catch (InterruptedException e) { - fail("sleep interrupted"); - } + thriftClientPool.close(); - // Verify client is destroyed when abandoned - verify(mockClient.getInputProtocol().getTransport(), times(1)).close(); - verify(mockClient.getOutputProtocol().getTransport(), times(1)).close(); + // Verify client is destroyed when pool is closed + verify(mockInputTransport, times(1)).close(); + verify(mockOutputTransport, times(1)).close(); } - - @org.springframework.context.annotation.Configuration - @ComponentScan( - basePackages = {"org.apache.airavata.api", "org.apache.airavata.config"}, - excludeFilters = { - @org.springframework.context.annotation.ComponentScan.Filter( - type = org.springframework.context.annotation.FilterType.ASSIGNABLE_TYPE, - classes = { - org.apache.airavata.config.BackgroundServicesLauncher.class, - org.apache.airavata.config.ThriftServerLauncher.class - }) - }) - @Import(org.apache.airavata.config.AiravataPropertiesConfiguration.class) - static class TestConfiguration {} } diff --git a/airavata-api/src/test/java/org/apache/airavata/config/SchemaValidationTest.java b/airavata-api/src/test/java/org/apache/airavata/config/SchemaValidationTest.java index 721bc5f036..823b3fc2a4 100644 --- a/airavata-api/src/test/java/org/apache/airavata/config/SchemaValidationTest.java +++ b/airavata-api/src/test/java/org/apache/airavata/config/SchemaValidationTest.java @@ -19,6 +19,9 @@ */ package org.apache.airavata.config; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.fail; + import jakarta.persistence.EntityManagerFactory; import jakarta.persistence.Persistence; import java.sql.Connection; @@ -32,12 +35,9 @@ import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.fail; - /** * Test to validate that Hibernate entities match the database schema defined in ddl.sql. - * + * * This test: * 1. Loads the DDL SQL file * 2. Creates an in-memory H2 database with the schema from DDL @@ -69,7 +69,7 @@ public class SchemaValidationTest { } catch (ClassNotFoundException e) { throw new IllegalStateException("H2 database driver not found. Ensure h2 dependency is in test scope.", e); } - + // Create in-memory H2 database // Note: Hibernate validate mode will check entity mappings against the database schema. // For this test, we're primarily validating that: @@ -81,7 +81,7 @@ public class SchemaValidationTest { // set up with the schema from ddl.sql, and this test will validate entities match. jdbcUrl = "jdbc:h2:mem:schema_validation_test;DB_CLOSE_DELAY=-1;MODE=MySQL"; h2Connection = DriverManager.getConnection(jdbcUrl, "sa", ""); - + loadAndExecuteDDL(h2Connection); } @@ -94,25 +94,25 @@ public class SchemaValidationTest { /** * Set up database connection for schema validation. - * + * * Note: Hibernate's validate mode (hibernate.hbm2ddl.auto=validate) will: * - Check that tables exist (if schema is present) * - Validate column types match entity field types * - Verify foreign key constraints * - Check that required columns are not null - * + * * For full validation against ddl.sql: * 1. Set up a test database with the schema from ddl.sql * 2. Configure the test to use that database connection * 3. Hibernate validate mode will then check entities against the actual schema - * + * * This test validates entity structure and mapping correctness. * To validate against the full ddl.sql, ensure the database schema is set up first. */ private void loadAndExecuteDDL(Connection connection) throws SQLException { // For this test, we're primarily validating that: // 1. Entity classes are correctly annotated - // 2. Entity mappings are syntactically correct + // 2. Entity mappings are syntactically correct // 3. Hibernate can process entities without mapping errors // // Hibernate validate mode requires the schema to exist. @@ -120,7 +120,7 @@ public class SchemaValidationTest { // - Set up a test database with the schema from ddl.sql // - Update the jdbcUrl to point to that database // - Hibernate will then validate entities against the actual schema - + logger.info("Schema validation test initialized"); logger.info("Using Hibernate validate mode - entities will be validated against database schema"); logger.info("For full ddl.sql validation, ensure database schema matches ddl.sql"); @@ -173,7 +173,7 @@ public class SchemaValidationTest { logger.error("✗ Persistence unit '{}' validation failed: {}", puName, e.getMessage()); } } - + if (failures > 0) { fail(String.format("Schema validation failed for %d persistence unit(s)", failures)); } @@ -181,13 +181,13 @@ public class SchemaValidationTest { /** * Validate a persistence unit against the database schema. - * + * * Hibernate's validate mode checks: * - Entity mappings are syntactically correct * - Tables exist (if schema is present) * - Column types match * - Foreign keys are correctly defined - * + * * Note: For full validation against ddl.sql, ensure the database schema * matches the DDL. This test validates entity structure and mapping correctness. */ @@ -200,7 +200,7 @@ public class SchemaValidationTest { properties.put("jakarta.persistence.jdbc.user", "sa"); properties.put("jakarta.persistence.jdbc.password", ""); properties.put("hibernate.dialect", "org.hibernate.dialect.H2Dialect"); - + // Use update mode to create schema from entities // This validates that: // 1. Entity mappings are syntactically correct @@ -212,35 +212,35 @@ public class SchemaValidationTest { // 2. Change hbm2ddl.auto to "validate" // 3. Hibernate will then check entities match the actual schema properties.put("hibernate.hbm2ddl.auto", "update"); - + // For better error messages properties.put("hibernate.show_sql", "false"); properties.put("hibernate.format_sql", "false"); - + logger.info("Validating persistence unit: {}", persistenceUnitName); - + // This will throw an exception if: // - Entity mappings are invalid // - Schema validation fails (tables/columns don't match) // - Entity classes have errors emf = Persistence.createEntityManagerFactory(persistenceUnitName, properties); - + assertNotNull(emf, "EntityManagerFactory should be created"); assertNotNull(emf.getMetamodel(), "Metamodel should be available"); - + // Try to access the metamodel to ensure it's fully initialized var entities = emf.getMetamodel().getEntities(); assertNotNull(entities, "Entities should be available in metamodel"); - - logger.info("✓ Persistence unit '{}' schema validation passed ({} entities)", - persistenceUnitName, entities.size()); - + + logger.info( + "✓ Persistence unit '{}' schema validation passed ({} entities)", + persistenceUnitName, + entities.size()); + } catch (jakarta.persistence.PersistenceException e) { // Hibernate validation errors are wrapped in PersistenceException String errorMsg = String.format( - "Schema validation failed for persistence unit '%s': %s", - persistenceUnitName, - e.getMessage()); + "Schema validation failed for persistence unit '%s': %s", persistenceUnitName, e.getMessage()); if (e.getCause() != null) { errorMsg += " (Caused by: " + e.getCause().getMessage() + ")"; } @@ -248,9 +248,7 @@ public class SchemaValidationTest { fail(errorMsg, e); } catch (Exception e) { String errorMsg = String.format( - "Unexpected error validating persistence unit '%s': %s", - persistenceUnitName, - e.getMessage()); + "Unexpected error validating persistence unit '%s': %s", persistenceUnitName, e.getMessage()); logger.error(errorMsg, e); fail(errorMsg, e); } finally { @@ -260,4 +258,3 @@ public class SchemaValidationTest { } } } - diff --git a/airavata-api/src/test/java/org/apache/airavata/security/KeyCloakSecurityManagerTest.java b/airavata-api/src/test/java/org/apache/airavata/security/KeyCloakSecurityManagerTest.java index ffd7e1a67b..cc4c0b4da1 100644 --- a/airavata-api/src/test/java/org/apache/airavata/security/KeyCloakSecurityManagerTest.java +++ b/airavata-api/src/test/java/org/apache/airavata/security/KeyCloakSecurityManagerTest.java @@ -41,6 +41,7 @@ import org.apache.airavata.security.authzcache.AuthzCachedStatus; import org.apache.airavata.security.model.AuthzToken; import org.apache.airavata.service.SharingRegistryService; import org.apache.airavata.service.registry.RegistryService; +import org.apache.airavata.service.security.CredentialStoreService; import org.apache.airavata.sharing.model.UserGroup; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -49,6 +50,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.FilterType; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Primary; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.bean.override.mockito.MockitoBean; @@ -73,11 +75,11 @@ public class KeyCloakSecurityManagerTest { @MockitoBean private AuthzCacheManager mockAuthzCacheManager; - private final KeyCloakSecurityManager keyCloakSecurityManager; + @MockitoBean + private CredentialStoreService mockCredentialStoreService; - public KeyCloakSecurityManagerTest(KeyCloakSecurityManager keyCloakSecurityManager) { - this.keyCloakSecurityManager = keyCloakSecurityManager; - } + @Autowired + private KeyCloakSecurityManager keyCloakSecurityManager; @Configuration @ComponentScan( diff --git a/airavata-api/src/test/java/org/apache/airavata/model/util/GroupComputeResourcePreferenceUtilTest.java b/airavata-api/src/test/java/org/apache/airavata/util/GroupComputeResourcePreferenceUtilTest.java similarity index 98% rename from airavata-api/src/test/java/org/apache/airavata/model/util/GroupComputeResourcePreferenceUtilTest.java rename to airavata-api/src/test/java/org/apache/airavata/util/GroupComputeResourcePreferenceUtilTest.java index e1c8983506..7be133ed8d 100644 --- a/airavata-api/src/test/java/org/apache/airavata/model/util/GroupComputeResourcePreferenceUtilTest.java +++ b/airavata-api/src/test/java/org/apache/airavata/util/GroupComputeResourcePreferenceUtilTest.java @@ -17,13 +17,14 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.airavata.model.util; +package org.apache.airavata.util; import java.util.Arrays; import java.util.Collections; import java.util.List; import org.apache.airavata.common.model.ComputeResourceReservation; import org.apache.airavata.common.model.GroupComputeResourcePreference; +import org.apache.airavata.util.GroupComputeResourcePreferenceUtil; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; diff --git a/modules/agent-framework/agent-service/src/main/java/org/apache/airavata/agent/connection/service/db/entity/AgentExecutionStatus.java b/modules/agent-framework/agent-service/src/main/java/org/apache/airavata/agent/connection/service/db/entity/AgentExecutionStatus.java index 0527542eab..196afcdc58 100644 --- a/modules/agent-framework/agent-service/src/main/java/org/apache/airavata/agent/connection/service/db/entity/AgentExecutionStatus.java +++ b/modules/agent-framework/agent-service/src/main/java/org/apache/airavata/agent/connection/service/db/entity/AgentExecutionStatus.java @@ -19,16 +19,14 @@ */ package org.apache.airavata.agent.connection.service.db.entity; -import org.hibernate.annotations.GenericGenerator; - +import jakarta.persistence.CascadeType; import jakarta.persistence.Column; import jakarta.persistence.Entity; - -import jakarta.persistence.CascadeType; import jakarta.persistence.FetchType; import jakarta.persistence.GeneratedValue; import jakarta.persistence.Id; import jakarta.persistence.ManyToOne; +import org.hibernate.annotations.GenericGenerator; @Entity(name = "AGENT_EXECUTION_STATUS") public class AgentExecutionStatus { diff --git a/modules/agent-framework/agent-service/src/main/java/org/apache/airavata/agent/connection/service/handlers/AgentConnectionHandler.java b/modules/agent-framework/agent-service/src/main/java/org/apache/airavata/agent/connection/service/handlers/AgentConnectionHandler.java index 775383b9d4..868f3936f6 100644 --- a/modules/agent-framework/agent-service/src/main/java/org/apache/airavata/agent/connection/service/handlers/AgentConnectionHandler.java +++ b/modules/agent-framework/agent-service/src/main/java/org/apache/airavata/agent/connection/service/handlers/AgentConnectionHandler.java @@ -27,7 +27,6 @@ import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; import net.devh.boot.grpc.server.service.GrpcService; - import org.apache.airavata.agent.AgentCommunicationServiceGrpc; import org.apache.airavata.agent.AgentMessage; import org.apache.airavata.agent.AgentPing; diff --git a/modules/agent-framework/agent-service/src/main/java/org/apache/airavata/agent/connection/service/handlers/FuseFSHandler.java b/modules/agent-framework/agent-service/src/main/java/org/apache/airavata/agent/connection/service/handlers/FuseFSHandler.java index 537cf57b83..8d491512aa 100644 --- a/modules/agent-framework/agent-service/src/main/java/org/apache/airavata/agent/connection/service/handlers/FuseFSHandler.java +++ b/modules/agent-framework/agent-service/src/main/java/org/apache/airavata/agent/connection/service/handlers/FuseFSHandler.java @@ -24,30 +24,30 @@ import com.google.protobuf.Timestamp; import io.grpc.stub.StreamObserver; import java.nio.charset.Charset; import net.devh.boot.grpc.server.service.GrpcService; -import org.apache.airavata.fuse.FuseServiceGrpc; -import org.apache.airavata.fuse.StatFsReq; -import org.apache.airavata.fuse.StatFsRes; -import org.apache.airavata.fuse.StatFs; +import org.apache.airavata.fuse.DirEntry; +import org.apache.airavata.fuse.FileEntry; +import org.apache.airavata.fuse.FileInfo; import org.apache.airavata.fuse.FileInfoReq; import org.apache.airavata.fuse.FileInfoRes; -import org.apache.airavata.fuse.FileInfo; +import org.apache.airavata.fuse.FuseServiceGrpc; +import org.apache.airavata.fuse.InodeAtt; import org.apache.airavata.fuse.OpenDirReq; import org.apache.airavata.fuse.OpenDirRes; -import org.apache.airavata.fuse.OpenedDir; import org.apache.airavata.fuse.OpenFileReq; import org.apache.airavata.fuse.OpenFileRes; +import org.apache.airavata.fuse.OpenedDir; import org.apache.airavata.fuse.OpenedFile; import org.apache.airavata.fuse.ReadDirReq; import org.apache.airavata.fuse.ReadDirRes; -import org.apache.airavata.fuse.DirEntry; import org.apache.airavata.fuse.ReadFileReq; import org.apache.airavata.fuse.ReadFileRes; -import org.apache.airavata.fuse.FileEntry; -import org.apache.airavata.fuse.WriteFileReq; -import org.apache.airavata.fuse.WriteFileRes; import org.apache.airavata.fuse.SetInodeAttReq; import org.apache.airavata.fuse.SetInodeAttRes; -import org.apache.airavata.fuse.InodeAtt; +import org.apache.airavata.fuse.StatFs; +import org.apache.airavata.fuse.StatFsReq; +import org.apache.airavata.fuse.StatFsRes; +import org.apache.airavata.fuse.WriteFileReq; +import org.apache.airavata.fuse.WriteFileRes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/modules/registry-db-migrator/src/main/java/org/apache/airavata/registry/tool/DBMigrator.java b/modules/registry-db-migrator/src/main/java/org/apache/airavata/registry/tool/DBMigrator.java index 908be800be..f471a7cfdf 100644 --- a/modules/registry-db-migrator/src/main/java/org/apache/airavata/registry/tool/DBMigrator.java +++ b/modules/registry-db-migrator/src/main/java/org/apache/airavata/registry/tool/DBMigrator.java @@ -39,12 +39,11 @@ import java.util.Calendar; import java.util.Date; import java.util.Scanner; import java.util.StringTokenizer; - import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.DefaultParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory;
