Repository: airavata Updated Branches: refs/heads/lahiru/AIRAVATA-2065 415e9b701 -> 46b8502eb
First stage of kafka appender Project: http://git-wip-us.apache.org/repos/asf/airavata/repo Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/46b8502e Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/46b8502e Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/46b8502e Branch: refs/heads/lahiru/AIRAVATA-2065 Commit: 46b8502ebe3d2c76000db8d79329e58434810ef3 Parents: 415e9b7 Author: Lahiru Ginnaliya Gamathige <[email protected]> Authored: Sun Aug 28 21:39:11 2016 -0700 Committer: Lahiru Ginnaliya Gamathige <[email protected]> Committed: Sun Aug 28 21:39:11 2016 -0700 ---------------------------------------------------------------------- distribution/pom.xml | 11 +- modules/commons/pom.xml | 12 +- .../airavata/common/logging/Exception.java | 66 +++++++++ .../airavata/common/logging/GsonTest.java | 20 +++ .../airavata/common/logging/LogEntry.java | 132 ++++++++++++++++++ .../airavata/common/logging/ServerId.java | 68 +++++++++ .../common/logging/kafka/KafkaAppender.java | 116 ++++++++++++++++ .../airavata/common/utils/AiravataZKUtils.java | 13 +- .../common/utils/ApplicationSettings.java | 34 +++-- .../airavata/common/utils/AwsMetadata.java | 137 +++++++++++++++++++ .../airavata/common/utils/BuildConstant.java | 28 ++++ .../apache/airavata/common/utils/JSONUtil.java | 1 - .../airavata/common/utils/ServerSettings.java | 36 +++++ .../main/resources/airavata-server.properties | 8 +- .../gfac/bes/provider/impl/BESProvider.java | 7 +- .../impl/JSDLGeneratorTestWithMyProxyAuth.java | 2 +- .../apache/airavata/gfac/core/GFacUtils.java | 2 +- .../gfac/impl/task/BESJobSubmissionTask.java | 7 +- .../services/impl/BigRed2TestWithSSHAuth.java | 2 +- .../impl/GSISSHProviderTestWithMyProxyAuth.java | 2 +- .../impl/SSHProviderTestWithSSHAuth.java | 2 +- modules/registry/registry-core/pom.xml | 27 ---- .../org/apache/airavata/server/ServerMain.java | 23 +++- .../utils/PropertyReader.java | 3 +- .../core/AbstractThriftDeserializer.java | 3 +- pom.xml | 32 +++++ 26 files changed, 728 insertions(+), 66 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/distribution/pom.xml ---------------------------------------------------------------------- diff --git a/distribution/pom.xml b/distribution/pom.xml index d3124eb..e3d1b33 100644 --- a/distribution/pom.xml +++ b/distribution/pom.xml @@ -559,7 +559,16 @@ <artifactId>curator-framework</artifactId> <version>${curator.version}</version> </dependency> - + <dependency> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka-clients</artifactId> + <version>${kafka-clients.version}</version> + </dependency> + <dependency> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-classic</artifactId> + <version>${logback.version}</version> + </dependency> <!-- ======================== Sample =================== --> <dependency> <groupId>org.apache.airavata</groupId> http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/pom.xml ---------------------------------------------------------------------- diff --git a/modules/commons/pom.xml b/modules/commons/pom.xml index a2136c6..f4231ce 100644 --- a/modules/commons/pom.xml +++ b/modules/commons/pom.xml @@ -115,7 +115,6 @@ <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> - <scope>test</scope> </dependency> <dependency> <groupId>org.apache.curator</groupId> @@ -127,12 +126,21 @@ <artifactId>libthrift</artifactId> <version>${thrift.version}</version> </dependency> - <dependency> <groupId>com.google.code.gson</groupId> <artifactId>gson</artifactId> <version>${google.gson.version}</version> </dependency> + <dependency> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka-clients</artifactId> + <version>${kafka-clients.version}</version> + </dependency> + <dependency> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-classic</artifactId> + <version>${logback.version}</version> + </dependency> </dependencies> <properties> http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/logging/Exception.java ---------------------------------------------------------------------- diff --git a/modules/commons/src/main/java/org/apache/airavata/common/logging/Exception.java b/modules/commons/src/main/java/org/apache/airavata/common/logging/Exception.java new file mode 100644 index 0000000..cea0c95 --- /dev/null +++ b/modules/commons/src/main/java/org/apache/airavata/common/logging/Exception.java @@ -0,0 +1,66 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.common.logging; + + +public class Exception { + + private String message; + + private String[] stackTrace; + + private String className; + + public Exception(String message, String[] stackTrace) { + this.message = message; + this.stackTrace = stackTrace; + } + + public Exception(String message, String[] stackTrace, String className) { + this.message = message; + this.stackTrace = stackTrace; + this.className = className; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public String[] getStackTrace() { + return stackTrace; + } + + public void setStackTrace(String[] stackTrace) { + this.stackTrace = stackTrace; + } + + public String getClassName() { + return className; + } + + public void setClassName(String className) { + this.className = className; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/logging/GsonTest.java ---------------------------------------------------------------------- diff --git a/modules/commons/src/main/java/org/apache/airavata/common/logging/GsonTest.java b/modules/commons/src/main/java/org/apache/airavata/common/logging/GsonTest.java new file mode 100644 index 0000000..b4e9cb2 --- /dev/null +++ b/modules/commons/src/main/java/org/apache/airavata/common/logging/GsonTest.java @@ -0,0 +1,20 @@ +/// +// Copyright (c) 2016. Highfive Technologies, Inc. +/// +package org.apache.airavata.common.logging; + +import com.google.gson.Gson; + +import java.util.HashMap; + +public class GsonTest { + public static void main(String[] args) { + final String[] strings = {"roles"}; + final Exception exception = new Exception("message", strings, "test"); + final LogEntry logEntry = new LogEntry(new ServerId("serverId", "serverId", "serverId", strings), "serverId", "serverId", "serverId", "serverId", new HashMap<String, String>(), "thread"); + final LogEntry logEntry1 = new LogEntry(new ServerId("serverId", "serverId", "serverId", strings), "serverId", "serverId", "serverId", "serverId", new HashMap<String, String>(), "thread", exception); + + System.out.println(new Gson().toJson(logEntry)); + System.out.println(new Gson().toJson(logEntry1)); + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/logging/LogEntry.java ---------------------------------------------------------------------- diff --git a/modules/commons/src/main/java/org/apache/airavata/common/logging/LogEntry.java b/modules/commons/src/main/java/org/apache/airavata/common/logging/LogEntry.java new file mode 100644 index 0000000..72fc4a0 --- /dev/null +++ b/modules/commons/src/main/java/org/apache/airavata/common/logging/LogEntry.java @@ -0,0 +1,132 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.common.logging; + + +import java.lang.*; +import java.util.Map; + +public class LogEntry { + + private ServerId serverId; + + private String message; + + private String timestamp; + + private String level; + + private String loggerName; + + private Map<String, String> mdc; + + private String threadName; + + private Exception exception; + + public LogEntry(ServerId serverId, String message, String timestamp, String level, String loggerName, Map<String, + String> mdc, String threadName, Exception exception) { + this.serverId = serverId; + this.message = message; + this.timestamp = timestamp; + this.level = level; + this.loggerName = loggerName; + this.mdc = mdc; + this.threadName = threadName; + this.exception = exception; + } + + public LogEntry(ServerId serverId, String message, String timestamp, String level, String loggerName, Map<String, + String> mdc, String threadName) { + this.serverId = serverId; + this.message = message; + this.timestamp = timestamp; + this.level = level; + this.loggerName = loggerName; + this.mdc = mdc; + this.threadName = threadName; + } + + + public ServerId getServerId() { + return serverId; + } + + public void setServerId(ServerId serverId) { + this.serverId = serverId; + } + + public String getMessage() { + return message; + } + + public void setMessage(String message) { + this.message = message; + } + + public String getTimestamp() { + return timestamp; + } + + public void setTimestamp(String timestamp) { + this.timestamp = timestamp; + } + + public String getLevel() { + return level; + } + + public void setLevel(String level) { + this.level = level; + } + + public String getLoggerName() { + return loggerName; + } + + public void setLoggerName(String loggerName) { + this.loggerName = loggerName; + } + + public Map<String, String> getMdc() { + return mdc; + } + + public void setMdc(Map<String, String> mdc) { + this.mdc = mdc; + } + + public String getThreadName() { + return threadName; + } + + public void setThreadName(String threadName) { + this.threadName = threadName; + } + + public Exception getException() { + return exception; + } + + public void setException(Exception exception) { + this.exception = exception; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/logging/ServerId.java ---------------------------------------------------------------------- diff --git a/modules/commons/src/main/java/org/apache/airavata/common/logging/ServerId.java b/modules/commons/src/main/java/org/apache/airavata/common/logging/ServerId.java new file mode 100644 index 0000000..9611302 --- /dev/null +++ b/modules/commons/src/main/java/org/apache/airavata/common/logging/ServerId.java @@ -0,0 +1,68 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.common.logging; + +public class ServerId { + private String serverId; + private String hostName; + private String version; + + private String[] roles; // gfac, orchestrator, apiserver, + + public ServerId(String serverId, String hostName, String version, String[] roles) { + this.serverId = serverId; + this.hostName = hostName; + this.version = version; + this.roles = roles; + } + + public String getServerId() { + return serverId; + } + + public void setServerId(String serverId) { + this.serverId = serverId; + } + + public String getHostName() { + return hostName; + } + + public void setHostName(String hostName) { + this.hostName = hostName; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public String[] getRoles() { + return roles; + } + + public void setRoles(String[] roles) { + this.roles = roles; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/logging/kafka/KafkaAppender.java ---------------------------------------------------------------------- diff --git a/modules/commons/src/main/java/org/apache/airavata/common/logging/kafka/KafkaAppender.java b/modules/commons/src/main/java/org/apache/airavata/common/logging/kafka/KafkaAppender.java new file mode 100644 index 0000000..a4a6a7b --- /dev/null +++ b/modules/commons/src/main/java/org/apache/airavata/common/logging/kafka/KafkaAppender.java @@ -0,0 +1,116 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.common.logging.kafka; + + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.IThrowableProxy; +import ch.qos.logback.classic.spi.StackTraceElementProxy; +import ch.qos.logback.core.UnsynchronizedAppenderBase; +import com.google.gson.Gson; +import org.apache.airavata.common.logging.Exception; +import org.apache.airavata.common.logging.LogEntry; +import org.apache.airavata.common.logging.ServerId; +import org.apache.airavata.common.utils.AwsMetadata; +import org.apache.airavata.common.utils.BuildConstant; +import org.apache.airavata.common.utils.ServerSettings; +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.Producer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.util.Arrays; +import java.util.Properties; + +public class KafkaAppender extends UnsynchronizedAppenderBase<ILoggingEvent> { + private final static Logger logger = LoggerFactory.getLogger(KafkaAppender.class); + + private final Producer<String, String> producer; + private final String kafkaTopic; + + private ServerId serverId = null; + + public KafkaAppender(String kafkaTopicPrefix, String kafkaHost) { + Properties props = new Properties(); + props.put("bootstrap.servers", kafkaHost); + props.put("acks", "0"); + props.put("retries", 0); + props.put("batch.size", 16384); + props.put("linger.ms", 10000); // Send the batch every 5 seconds + props.put("buffer.memory", 33554432); + props.put("producer.type", "async"); + props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer"); + props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer"); + this.kafkaTopic = getKafkaTopicPrefix(kafkaTopicPrefix); + + logger.info("Starting kafka producer: bootstrap-server:{}, topic : {}", kafkaHost, kafkaTopic); + this.producer = new KafkaProducer<>(props); + if(ServerSettings.isRunningOnAws()) { + final AwsMetadata awsMetadata = new AwsMetadata(); + serverId = new ServerId(awsMetadata.getId(), awsMetadata.getHostname(), + BuildConstant.getGitDescribeVersion(), ServerSettings.getServerRoles()); + } else { + serverId = new ServerId(ServerSettings.getIp(), ServerSettings.getIp(), + BuildConstant.getGitDescribeVersion(), ServerSettings.getServerRoles()); + } + } + + @Override + protected void append(ILoggingEvent event) { + //todo do more elegant streaming approach to publish logs + if (!event.getLevel().equals(Level.ALL) && // OFF AND ALL are not loggable levels + !event.getLevel().equals(Level.OFF)) { + final IThrowableProxy throwableProxy = event.getThrowableProxy(); + final LogEntry entry = throwableProxy != null ? + new LogEntry(serverId, event.getMessage(), Instant.ofEpochMilli(event.getTimeStamp()).toString(), + event.getLevel().toString(), event.getLoggerName(), event.getMDCPropertyMap(), + event.getThreadName() != null ? event.getThreadName() : null, + new Exception(throwableProxy.getMessage(), toStringArray(throwableProxy.getStackTraceElementProxyArray()) + , throwableProxy.getClassName())) + : new LogEntry(serverId, event.getMessage(), Instant.ofEpochMilli(event.getTimeStamp()).toString(), + event.getLevel().toString(), event.getLoggerName(), event.getMDCPropertyMap(), + event.getThreadName() != null ? event.getThreadName() : null); + + producer.send(new ProducerRecord<>(kafkaTopic, new Gson().toJson(entry))); + } + } + + + private String[] toStringArray(StackTraceElementProxy[] stackTraceElement) { + return Arrays.stream(stackTraceElement).map(StackTraceElementProxy::getSTEAsString).toArray(String[]::new); + } + + private String getKafkaTopicPrefix(String kafkaTopicPrefix) { + final StringBuilder stringBuffer = new StringBuilder(""); + for (String role : ServerSettings.getServerRoles()) { + stringBuffer.append("_"); + stringBuffer.append(role); + } + return kafkaTopicPrefix + stringBuffer.toString(); + } + + public void close() { + producer.close(); + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java ---------------------------------------------------------------------- diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java index 5faf985..75f91fd 100644 --- a/modules/commons/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java +++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/AiravataZKUtils.java @@ -119,13 +119,14 @@ public class AiravataZKUtils implements Watcher { public static void startEmbeddedZK(ServerCnxnFactory cnxnFactory) { if (ServerSettings.isEmbeddedZK()) { ServerConfig serverConfig = new ServerConfig(); - URL resource = AiravataZKUtils.class.getClassLoader().getResource("zoo.cfg"); - if (resource == null) { - logger.error("There is no zoo.cfg file in the classpath... Failed to start Zookeeper Server"); - System.exit(1); - } + URL resource = ApplicationSettings.loadFile("zoo.cfg"); try { + if (resource == null) { + logger.error("There is no zoo.cfg file in the classpath... Failed to start Zookeeper Server"); + System.exit(1); + } serverConfig.parse(resource.getPath()); + } catch (QuorumPeerConfig.ConfigException e) { logger.error("Error while starting embedded Zookeeper", e); System.exit(2); @@ -193,7 +194,5 @@ public class AiravataZKUtils implements Watcher { } else { return bytesToLong(curatorClient.getData().storingStatIn(exists).forPath(cancelDeliveryTagPath)); } - - } } http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java ---------------------------------------------------------------------- diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java index 9ce0786..dc7944f 100644 --- a/modules/commons/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java +++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/ApplicationSettings.java @@ -25,9 +25,11 @@ import org.apache.airavata.common.exception.ApplicationSettingsException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; import java.net.URL; import java.util.*; import java.util.regex.Matcher; @@ -35,10 +37,12 @@ import java.util.regex.Pattern; public class ApplicationSettings { public static final String SERVER_PROPERTIES="airavata-server.properties"; - - public static String ADDITIONAL_SETTINGS_FILES = "external.settings"; + public static final String AIRAVATA_CONFIG_DIR = "airavata.config.dir"; + + public static String ADDITIONAL_SETTINGS_FILES = "external.settings"; protected Properties properties = new Properties(); + private Exception propertyLoadException; @@ -63,7 +67,6 @@ public class ApplicationSettings { private void loadProperties() { URL url = getPropertyFileURL(); try { - properties.load(url.openStream()); logger.info("Settings loaded from "+url.toString()); URL[] externalSettingsFileURLs = getExternalSettingsFileURLs(); @@ -77,7 +80,7 @@ public class ApplicationSettings { } protected URL getPropertyFileURL() { - return ApplicationSettings.class.getClassLoader().getResource(SERVER_PROPERTIES); + return ApplicationSettings.loadFile(SERVER_PROPERTIES); } protected URL[] getExternalSettingsFileURLs(){ @@ -86,7 +89,7 @@ public class ApplicationSettings { String externalSettingsFileNames = getSettingImpl(ADDITIONAL_SETTINGS_FILES); String[] externalSettingFiles = externalSettingsFileNames.split(","); for (String externalSettingFile : externalSettingFiles) { - URL externalSettingFileURL = ApplicationSettings.class.getClassLoader().getResource(externalSettingFile); + URL externalSettingFileURL = ApplicationSettings.loadFile(externalSettingFile); if (externalSettingFileURL==null){ logger.warn("Could not file external settings file "+externalSettingFile); }else{ @@ -313,11 +316,11 @@ public class ApplicationSettings { * Static methods which will be used by the users */ - public static String getSetting(String key) throws ApplicationSettingsException{ + public static String getSetting(String key) throws ApplicationSettingsException { return getInstance().getSettingImpl(key); } - - public static String getSetting(String key, String defaultValue){ + + public static String getSetting(String key, String defaultValue) { return getInstance().getSettingImpl(key,defaultValue); } @@ -426,4 +429,19 @@ public class ApplicationSettings { public static ShutdownStrategy getShutdownStrategy() throws Exception{ return getInstance().getShutdownStrategyImpl(); } + + public static URL loadFile(String fileName) { + final URL resource = ApplicationSettings.class.getClassLoader().getResource(fileName); + if(resource == null) { + if(System.getProperty(AIRAVATA_CONFIG_DIR) != null) { + final String airavataConfigDir = System.getProperty(AIRAVATA_CONFIG_DIR); + try { + return new File(airavataConfigDir + File.separator + fileName).toURI().toURL(); + } catch (MalformedURLException e) { + logger.error("Error parsing the file from airavata.config.dir", airavataConfigDir); + } + } + } + return resource; + } } http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/AwsMetadata.java ---------------------------------------------------------------------- diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/AwsMetadata.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/AwsMetadata.java new file mode 100644 index 0000000..f9b4a65 --- /dev/null +++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/AwsMetadata.java @@ -0,0 +1,137 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.common.utils; + + +import com.google.common.base.Preconditions; +import com.google.common.net.InetAddresses; +import com.google.gson.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.net.InetAddress; +import java.net.URI; +import java.net.URISyntaxException; + +public class AwsMetadata { + private static final Logger log = LoggerFactory.getLogger(AwsMetadata.class); + private static final String METADATA_URI_BASE = "http://169.254.169.254/";; + private static final String REGION_SUFFIX = "/latest/dynamic/instance-identity/document"; + private static final String ZONE_SUFFIX = "/latest/meta-data/placement/availability-zone"; + private static final String PUBLIC_IPV4_SUFFIX = "/latest/meta-data/public-ipv4"; + private static final String PRIVATE_IPV4_SUFFIX = "latest/meta-data/local-ipv4"; + private static final String HOSTNAME_SUFFIX = "/latest/meta-data/hostname"; + private static final String ID_SUFFIX = "/latest/meta-data/instance-id"; + + private final URI baseUri; + + private String id; + private String region; + private String hostname; + private String zone; + private InetAddress publicIp; + private InetAddress privateIp; + + public AwsMetadata() { + try { + baseUri = new URI(METADATA_URI_BASE); + } catch (URISyntaxException e) { + Preconditions.checkState(false, "Unexpected URI Syntax Exception: {}", e); + throw new RuntimeException(e); + } + } + + public String getRegion() { + if (region == null) { + try { + String dynamicData = getMetadataAt(REGION_SUFFIX); + if (dynamicData != null) { + final JsonObject asJsonObject = new JsonParser().parse(dynamicData).getAsJsonObject(); + region = asJsonObject.get("region").getAsString(); + } + } catch (ClassCastException e) { + log.error("Unable to get region, expecting a JSON Object", e); + } + } + return region; + } + + public String getZoneName() { + if (zone == null) { + zone = getMetadataAt(ZONE_SUFFIX); + } + return zone; + } + + public String getId() { + if (id == null) { + id = getMetadataAt(ID_SUFFIX); + } + + return id; + } + + public String getHostname() { + if (hostname == null) { + hostname = getMetadataAt(HOSTNAME_SUFFIX); + } + return hostname; + } + + public InetAddress getPublicIpAddress() { + if (publicIp == null) { + String ip = getMetadataAt(PUBLIC_IPV4_SUFFIX); + if (ip != null) { + publicIp = InetAddresses.forString(ip); + } + } + return publicIp; + } + + public InetAddress getInternalIpAddress() { + if (privateIp == null) { + String ip = getMetadataAt(PRIVATE_IPV4_SUFFIX); + if (ip != null) { + privateIp = InetAddresses.forString(ip); + } + } + return privateIp; + } + + private String getMetadataAt(String suffix) { + try { + URI resolved = baseUri.resolve(suffix); + StringBuilder builder = new StringBuilder(); + String line = null; + try (BufferedReader reader = new BufferedReader(new InputStreamReader(resolved.toURL().openStream()))) { + while ((line = reader.readLine()) != null) { + builder.append(line); + } + return builder.toString(); + } + } catch (Exception e) { + // ignore for now to make sure local servers don't go verbose + } + return null; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/BuildConstant.java ---------------------------------------------------------------------- diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/BuildConstant.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/BuildConstant.java new file mode 100644 index 0000000..5598974 --- /dev/null +++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/BuildConstant.java @@ -0,0 +1,28 @@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.airavata.common.utils; + +public class BuildConstant { + public static final String GIT_DESCRIBE_VERSION = "airavata-0.16-128-g415e9b7"; + public static String getGitDescribeVersion() { + return GIT_DESCRIBE_VERSION; + } +} http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/JSONUtil.java ---------------------------------------------------------------------- diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/JSONUtil.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/JSONUtil.java index 222e5a2..59d065d 100644 --- a/modules/commons/src/main/java/org/apache/airavata/common/utils/JSONUtil.java +++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/JSONUtil.java @@ -28,7 +28,6 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.gson.JsonPrimitive; -import com.google.gson.stream.JsonReader; import java.io.File; import java.io.FileReader; http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java ---------------------------------------------------------------------- diff --git a/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java b/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java index 985daad..a14cb9d 100644 --- a/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java +++ b/modules/commons/src/main/java/org/apache/airavata/common/utils/ServerSettings.java @@ -27,6 +27,8 @@ import org.slf4j.LoggerFactory; import java.net.InetAddress; import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.Map; public class ServerSettings extends ApplicationSettings { @@ -107,6 +109,15 @@ public class ServerSettings extends ApplicationSettings { private static final String EMAIL_BASED_MONITOR_STORE_PROTOCOL = "email.based.monitor.store.protocol"; private static final String ENABLE_EMAIL_BASED_MONITORING = "enable.email.based.monitoring"; + private static final String IS_RUNNING_ON_AWS = "isRunningOnAws"; + private static final String ENABLE_KAFKA_LOGGING = "enable.kafka.logging"; + private static final String KAFKA_BROKER_LIST = "kafka.broker.list"; + private static final String KAFKA_TOPIC_PREFIX = "kafka.topic.prefix"; + private static final String SERVER_ROLES = "server.roles"; + + // todo until AIRAVATA-2066 is finished, keep server side list configurations here. + private static Map<String, String[]> listConfigurations = new HashMap<>(); + private static boolean stopAllThreads = false; private static boolean emailBaseNotificationEnable; private static String outputLocation; @@ -400,4 +411,29 @@ public class ServerSettings extends ApplicationSettings { public static String getLocalDataLocation() { return System.getProperty("java.io.tmpdir"); } + + public static boolean isRunningOnAws() { + return Boolean.valueOf(getSetting(IS_RUNNING_ON_AWS, "false")); + } + + public static String getKafkaBrokerList() { + return getSetting(KAFKA_BROKER_LIST, null); + } + + public static String getKafkaTopicPrefix() { + return getSetting(KAFKA_TOPIC_PREFIX, "all"); + } + + public static boolean isEnabledKafkaLogging() { + return Boolean.valueOf(getSetting(ENABLE_KAFKA_LOGGING, "false")); + } + + public static void setServerRoles(String[] roles) { + listConfigurations.put(SERVER_ROLES, roles); + } + + public static String[] getServerRoles() { + return listConfigurations.get(SERVER_ROLES); + } + } http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/configuration/server/src/main/resources/airavata-server.properties ---------------------------------------------------------------------- diff --git a/modules/configuration/server/src/main/resources/airavata-server.properties b/modules/configuration/server/src/main/resources/airavata-server.properties index 0dad713..d079a6f 100644 --- a/modules/configuration/server/src/main/resources/airavata-server.properties +++ b/modules/configuration/server/src/main/resources/airavata-server.properties @@ -283,4 +283,10 @@ authorization.policy=airavata-default-xacml-policy #### authorization cache related configuration #### authz.cache.enabled=true authz.cache.manager.class=org.apache.airavata.api.server.security.authzcache.DefaultAuthzCacheManager -in.memory.cache.size=1000 \ No newline at end of file +in.memory.cache.size=1000 + +# Kafka Logging related configuration +isRunningOnAws= false +kafka.broker.list= 54.84.144.132:9092 +kafka.topic.prefix= local +enable.kafka.logging= true http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java index b287b8e..efd2c5f 100644 --- a/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java +++ b/modules/gfac/gfac-bes/src/main/java/org/apache/airavata/gfac/bes/provider/impl/BESProvider.java @@ -144,10 +144,9 @@ public class BESProvider extends AbstractProvider implements GFacProvider, JobDetails jobDetails = new JobDetails(); FactoryClient factory = new FactoryClient(eprt, secProperties); - log.info(String.format("Activity Submitting to %s ... \n", - factoryUrl)); + log.info("Activity Submitting to {} ... \n", factoryUrl); CreateActivityResponseDocument response = factory.createActivity(cad); - log.info(String.format("Activity Submitted to %s \n", factoryUrl)); + log.info("Activity Submitted to {} \n", factoryUrl); EndpointReferenceType activityEpr = response.getCreateActivityResponse().getActivityIdentifier(); @@ -353,7 +352,7 @@ public class BESProvider extends AbstractProvider implements GFacProvider, } protected String formatStatusMessage(String activityUrl, String status) { - return String.format("Activity %s is %s.\n", activityUrl, status); + return String.format("Activity {} is {}.\n", activityUrl, status); } protected String subStatusAsString(ActivityStatusType statusType) { http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java b/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java index cf9b82a..90f67a1 100644 --- a/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java +++ b/modules/gfac/gfac-bes/src/test/java/org/apache/airavata/core/gfac/services/impl/JSDLGeneratorTestWithMyProxyAuth.java @@ -106,7 +106,7 @@ import static org.junit.Assert.assertTrue; // } // // protected GFacConfiguration getGFACConfig() throws Exception{ -// URL resource = BESProviderTestWithMyProxyAuth.class.getClassLoader().getResource(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML); +// URL resource = ApplicationSettings.loadFile(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML); // System.out.println(resource.getFile()); // GFacConfiguration gFacConfiguration = GFacConfiguration.create(new File(resource.getPath()),null,null); // return gFacConfiguration; http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java index 0ed836f..545e122 100644 --- a/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java +++ b/modules/gfac/gfac-core/src/main/java/org/apache/airavata/gfac/core/GFacUtils.java @@ -801,7 +801,7 @@ public class GFacUtils { public static File createJobFile(TaskContext taskContext, JobDescriptor jobDescriptor, JobManagerConfiguration jobManagerConfiguration) throws GFacException { try { TransformerFactory factory = TransformerFactory.newInstance(); - URL resource = GFacUtils.class.getClassLoader().getResource(jobManagerConfiguration.getJobDescriptionTemplateName()); + URL resource = ApplicationSettings.loadFile(jobManagerConfiguration.getJobDescriptionTemplateName()); if (resource == null) { String error = "System configuration file '" + jobManagerConfiguration.getJobDescriptionTemplateName() http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java index 6529395..1955225 100644 --- a/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java +++ b/modules/gfac/gfac-impl/src/main/java/org/apache/airavata/gfac/impl/task/BESJobSubmissionTask.java @@ -151,10 +151,9 @@ public class BESJobSubmissionTask implements JobSubmissionTask { jobDetails.setProcessId(taskContext.getProcessId()); FactoryClient factory = new FactoryClient(eprt, secProperties); - log.info(String.format("Activity Submitting to %s ... \n", - factoryUrl)); + log.info("Activity Submitting to {} ... \n", factoryUrl); CreateActivityResponseDocument response = factory.createActivity(cad); - log.info(String.format("Activity Submitted to %s \n", factoryUrl)); + log.info("Activity Submitted to {} \n", factoryUrl); EndpointReferenceType activityEpr = response.getCreateActivityResponse().getActivityIdentifier(); @@ -356,7 +355,7 @@ public class BESJobSubmissionTask implements JobSubmissionTask { } protected String formatStatusMessage(String activityUrl, String status) { - return String.format("Activity %s is %s.\n", activityUrl, status); + return String.format("Activity {} is {}.\n", activityUrl, status); } protected void waitUntilDone(EndpointReferenceType factoryEpr, EndpointReferenceType activityEpr, ProcessContext processContext, DefaultClientConfiguration secProperties) throws Exception { http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java index 4aecd57..fbad51c 100644 --- a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java +++ b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/BigRed2TestWithSSHAuth.java @@ -91,7 +91,7 @@ // System.out.println("Use -Dssh.username=xxx -Dssh.password=yyy -Dssh.keypass=zzz " + // "-Dprivate.ssh.key -Dpublic.ssh.key -Dssh.working.directory "); // } -// URL resource = BigRed2TestWithSSHAuth.class.getClassLoader().getResource(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML); +// URL resource = ApplicationSettings.loadFile(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML); // assert resource != null; // System.out.println(resource.getFile()); // GFacConfiguration gFacConfiguration = GFacConfiguration.create(new File(resource.getPath()), null); http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java index 4ca5684..9d352a4 100644 --- a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java +++ b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/GSISSHProviderTestWithMyProxyAuth.java @@ -90,7 +90,7 @@ // "E.g :- mvn clean install -Dmyproxy.username=xxx -Dmyproxy.password=xxx -Dgsi.working.directory=/path<<<<<<<"); // throw new Exception("Need my proxy user name password to run tests."); // } -// URL resource = GSISSHProviderTestWithMyProxyAuth.class.getClassLoader().getResource(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML); +// URL resource = ApplicationSettings.loadFile(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML); // assert resource != null; // System.out.println(resource.getFile()); // GFacConfiguration gFacConfiguration = GFacConfiguration.create(new File(resource.getPath()), null); http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java ---------------------------------------------------------------------- diff --git a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java index 4aa0df1..bdbadda 100644 --- a/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java +++ b/modules/gfac/gfac-impl/src/test/java/org/apache/airavata/core/gfac/services/impl/SSHProviderTestWithSSHAuth.java @@ -54,7 +54,7 @@ // @Before // public void setUp() throws Exception { // -// URL resource = SSHProviderTestWithSSHAuth.class.getClassLoader().getResource(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML); +// URL resource = ApplicationSettings.loadFile(org.apache.airavata.common.utils.Constants.GFAC_CONFIG_XML); // GFacConfiguration gFacConfiguration = GFacConfiguration.create(new File(resource.getPath()),null); //// gFacConfiguration.s // //have to set InFlwo Handlers and outFlowHandlers http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/registry/registry-core/pom.xml ---------------------------------------------------------------------- diff --git a/modules/registry/registry-core/pom.xml b/modules/registry/registry-core/pom.xml index 85986ae..60da666 100644 --- a/modules/registry/registry-core/pom.xml +++ b/modules/registry/registry-core/pom.xml @@ -107,33 +107,6 @@ <build> <plugins> - <plugin> - <groupId>org.apache.openjpa</groupId> - <artifactId>openjpa-maven-plugin</artifactId> - <version>2.2.0</version> - <configuration> - <includes>**/entities/*.class</includes> - <excludes>**/entities/XML*.class</excludes> - <addDefaultConstructor>true</addDefaultConstructor> - <enforcePropertyRestrictions>true</enforcePropertyRestrictions> - </configuration> - <executions> - <execution> - <id>enhancer</id> - <phase>process-classes</phase> - <goals> - <goal>enhance</goal> - </goals> - </execution> - </executions> - <dependencies> - <dependency> - <groupId>org.apache.openjpa</groupId> - <artifactId>openjpa</artifactId> - <version>2.2.0</version> - </dependency> - </dependencies> - </plugin> <!--<plugin>--> <!--<groupId>org.apache.maven.plugins</groupId>--> <!--<artifactId>maven-antrun-plugin</artifactId>--> http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/server/src/main/java/org/apache/airavata/server/ServerMain.java ---------------------------------------------------------------------- diff --git a/modules/server/src/main/java/org/apache/airavata/server/ServerMain.java b/modules/server/src/main/java/org/apache/airavata/server/ServerMain.java index 99387de..bcde1e9 100644 --- a/modules/server/src/main/java/org/apache/airavata/server/ServerMain.java +++ b/modules/server/src/main/java/org/apache/airavata/server/ServerMain.java @@ -20,13 +20,16 @@ */ package org.apache.airavata.server; +import ch.qos.logback.classic.LoggerContext; import org.apache.airavata.common.exception.ApplicationSettingsException; +import org.apache.airavata.common.logging.kafka.KafkaAppender; import org.apache.airavata.common.utils.*; import org.apache.airavata.common.utils.ApplicationSettings.ShutdownStrategy; import org.apache.airavata.common.utils.IServer.ServerStatus; import org.apache.airavata.common.utils.StringUtil.CommandLineParameters; import org.apache.commons.cli.ParseException; import org.apache.zookeeper.server.ServerCnxnFactory; +import org.slf4j.ILoggerFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -155,14 +158,29 @@ public class ServerMain { // } public static void main(String args[]) throws ParseException, IOException { - CommandLineParameters commandLineParameters = StringUtil.getCommandLineParser(args); + ServerSettings.mergeSettingsCommandLineArgs(args); + ServerSettings.setServerRoles(ApplicationSettings.getSetting(SERVERS_KEY, "all").split(",")); + + if (ServerSettings.isEnabledKafkaLogging()) { + final ILoggerFactory iLoggerFactory = LoggerFactory.getILoggerFactory(); + if (iLoggerFactory instanceof LoggerContext) { + final KafkaAppender kafkaAppender = new KafkaAppender(ServerSettings.getKafkaBrokerList(), + ServerSettings.getKafkaTopicPrefix()); + kafkaAppender.setContext((LoggerContext) iLoggerFactory); + kafkaAppender.setName("kafka-appender"); + kafkaAppender.clearAllFilters(); + kafkaAppender.start(); + ((LoggerContext) iLoggerFactory).getLogger(Logger.ROOT_LOGGER_NAME).addAppender(kafkaAppender); + } + } + + CommandLineParameters commandLineParameters = StringUtil.getCommandLineParser(args); if (commandLineParameters.getArguments().contains(STOP_COMMAND_STR)){ performServerStopRequest(commandLineParameters); }else{ AiravataZKUtils.startEmbeddedZK(cnxnFactory); performServerStart(args); } - } @@ -173,7 +191,6 @@ public class ServerMain { for (String string : args) { logger.info("Server Arguments: " + string); } - ServerSettings.mergeSettingsCommandLineArgs(args); String serverNames; try { serverNames = ApplicationSettings.getSetting(SERVERS_KEY); http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/utils/PropertyReader.java ---------------------------------------------------------------------- diff --git a/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/utils/PropertyReader.java b/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/utils/PropertyReader.java index f7b8e1f..9df1745 100644 --- a/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/utils/PropertyReader.java +++ b/modules/test-suite/multi-tenanted-airavata/src/main/java/org/apache/airavata/testsuite/multitenantedairavata/utils/PropertyReader.java @@ -21,6 +21,7 @@ package org.apache.airavata.testsuite.multitenantedairavata.utils; +import org.apache.airavata.common.utils.ApplicationSettings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,7 +42,7 @@ public class PropertyReader { } protected void loadProperties() throws IOException { - URL airavataURL = PropertyFileType.class.getClassLoader().getResource(TestFrameworkConstants.AIRAVATA_CLIENT_PROPERTIES); + URL airavataURL = ApplicationSettings.loadFile(TestFrameworkConstants.AIRAVATA_CLIENT_PROPERTIES); if (airavataURL != null){ airavataClientProperties.load(airavataURL.openStream()); } http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/modules/user-profile/user-profile-core/src/main/java/org/apache/airavata/userprofile/core/AbstractThriftDeserializer.java ---------------------------------------------------------------------- diff --git a/modules/user-profile/user-profile-core/src/main/java/org/apache/airavata/userprofile/core/AbstractThriftDeserializer.java b/modules/user-profile/user-profile-core/src/main/java/org/apache/airavata/userprofile/core/AbstractThriftDeserializer.java index 83af52b..14a4b9c 100644 --- a/modules/user-profile/user-profile-core/src/main/java/org/apache/airavata/userprofile/core/AbstractThriftDeserializer.java +++ b/modules/user-profile/user-profile-core/src/main/java/org/apache/airavata/userprofile/core/AbstractThriftDeserializer.java @@ -91,8 +91,7 @@ public abstract class AbstractThriftDeserializer<E extends TFieldIdEnum, T exten // Validate that the instance contains all required fields. validate(instance); } catch (final TException e) { - log.error(String.format("Unable to deserialize JSON '%s' to type '%s'.", - jp.getValueAsString(), instance.getClass().getName(), e)); + log.error("Unable to deserialize JSON {} to type {}.", jp.getValueAsString(), instance.getClass().getName(), e); ctxt.mappingException(e.getMessage()); } http://git-wip-us.apache.org/repos/asf/airavata/blob/46b8502e/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 6594026..00bc168 100644 --- a/pom.xml +++ b/pom.xml @@ -101,6 +101,8 @@ <snakeyaml.version>1.15</snakeyaml.version> <maven.javadoc.failOnError>false</maven.javadoc.failOnError> <maven.replacer.plugin.version>1.5.3</maven.replacer.plugin.version> + <kafka-clients.version>0.8.2.2</kafka-clients.version> + <logback.version>1.1.6</logback.version> </properties> <developers> @@ -434,6 +436,16 @@ <artifactId>curator-framework</artifactId> <version>${curator.version}</version> </dependency> + <dependency> + <groupId>org.apache.kafka</groupId> + <artifactId>kafka-clients</artifactId> + <version>${kafka-clients.version}</version> + </dependency> + <dependency> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-classic</artifactId> + <version>${logback.version}</version> + </dependency> </dependencies> </dependencyManagement> @@ -542,6 +554,26 @@ </excludes> </configuration> </plugin> + <plugin> + <groupId>com.lukegb.mojo</groupId> + <artifactId>gitdescribe-maven-plugin</artifactId> + <version>3.0</version> + <executions> + <execution> + <goals> + <goal>gitdescribe</goal> + </goals> + <id>git-describe</id> + <phase>initialize</phase> + <configuration> + <extraArguments> + <param>--tags</param> + </extraArguments> + <descriptionProperty>git-describe</descriptionProperty> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> <activation>
