http://git-wip-us.apache.org/repos/asf/aurora/blob/356eeac9/commons/src/main/java/org/apache/aurora/common/io/Base64ZlibCodec.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/io/Base64ZlibCodec.java b/commons/src/main/java/org/apache/aurora/common/io/Base64ZlibCodec.java deleted file mode 100644 index ed36e2a..0000000 --- a/commons/src/main/java/org/apache/aurora/common/io/Base64ZlibCodec.java +++ /dev/null @@ -1,169 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.common.io; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.UnsupportedEncodingException; -import java.io.Writer; -import java.nio.charset.Charset; -import java.util.zip.DeflaterOutputStream; -import java.util.zip.GZIPInputStream; -import java.util.zip.InflaterInputStream; - -import com.google.common.base.Preconditions; -import com.google.common.base.Throwables; -import com.google.common.io.ByteStreams; - -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.codec.binary.Base64OutputStream; - -/** - * Utility class providing encoding and decoding methods to and from a string to a utf-8 encoded, - * zlib compressed, Base64 encoded representation of the string. For wider compatibility, the - * decoder can also automatically recognize GZIP (instead of plain zlib) compressed data too and - * decode it accordingly. - * - * @author Attila Szegedi - */ -public final class Base64ZlibCodec { - /** - * Thrown to indicate invalid data while decoding or unzipping. - * - * @author Attila Szegedi - */ - public static class InvalidDataException extends Exception { - private static final long serialVersionUID = 1L; - - public InvalidDataException(String message) { - super(message); - } - - public InvalidDataException(String message, Throwable cause) { - super(message, cause); - } - } - - /** - * Text encoding used by the Base64 output stream. - */ - public static final String BASE64_TEXT_ENCODING = "ASCII"; - private static final int ESTIMATED_PLAINTEXT_TO_ENCODED_RATIO = 4; - - // Prefix all Base64-encoded, zlib compressed data must have - private static final byte[] ZLIB_HEADER_PREFIX = new byte[] { 120 }; - // Prefix all Base64-encoded, GZIP compressed data must have - private static final byte[] GZIP_HEADER_PREFIX = new byte[] {31, -117, 8, 0, 0, 0, 0, 0, 0 }; - private static final int DIAGNOSTIC_PREFIX_LENGTH = 16; - // Text encoding for char-to-byte transformation before compressing a stack trace - private static final Charset TEXT_ENCODING = com.google.common.base.Charsets.UTF_8; - - private Base64ZlibCodec() { - // Utility class - } - - /** - * Decodes a string. In addition to zlib, it also automatically detects GZIP compressed data and - * adjusts accordingly. - * - * @param encoded the encoded string, represented as a byte array of ASCII-encoded characters - * @return the decoded string - * @throws InvalidDataException if the string can not be decoded. - */ - public static byte[] decode(String encoded) throws InvalidDataException { - Preconditions.checkNotNull(encoded); - return decompress(new Base64().decode(encoded)); - } - - private static byte[] decompress(byte[] compressed) throws InvalidDataException { - byte[] bytes; - try { - final InputStream bin = new ByteArrayInputStream(compressed); - final InputStream zin; - if (startsWith(compressed, GZIP_HEADER_PREFIX)) { - zin = new GZIPInputStream(bin); - } else if (startsWith(compressed, ZLIB_HEADER_PREFIX)) { - zin = new InflaterInputStream(bin); - } else { - throw new Base64ZlibCodec.InvalidDataException("Value doesn't start with either GZIP or zlib header"); - } - try { - bytes = ByteStreams.toByteArray(zin); - } finally { - zin.close(); - } - } catch (IOException e) { - throw new Base64ZlibCodec.InvalidDataException("zlib/GZIP decoding error", e); - } - return bytes; - } - - private static boolean startsWith(byte[] value, byte[] prefix) { - final int pl = prefix.length; - if (value.length < pl) { - return false; - } - for (int i = 0; i < pl; ++i) { - if (value[i] != prefix[i]) { - return false; - } - } - return true; - } - - /** - * Encodes a set of bytes. - * - * @param plain the non-encoded bytes - * @return the encoded string - */ - public static String encode(byte[] plain) { - final ByteArrayOutputStream out = new ByteArrayOutputStream(plain.length - / ESTIMATED_PLAINTEXT_TO_ENCODED_RATIO); - final OutputStream w = getDeflatingEncodingStream(out); - try { - w.write(plain); - w.close(); - return out.toString(BASE64_TEXT_ENCODING); - } catch (UnsupportedEncodingException e) { - throw reportUnsupportedEncoding(); - } catch (IOException e) { - throw Throwables.propagate(e); - } - } - - private static OutputStream getDeflatingEncodingStream(OutputStream out) { - return new DeflaterOutputStream(new Base64OutputStream(out, true, - Integer.MAX_VALUE, null)); - } - - /** - * Returns a writer that writes through to the specified output stream, utf-8 encoding, - * zlib compressing, and Base64 encoding its input along the way. - * - * @param out the output stream that receives the final output - * @return a writer for the input - */ - public static Writer getEncodingWriter(OutputStream out) { - return new OutputStreamWriter(getDeflatingEncodingStream(out), TEXT_ENCODING); - } - - private static AssertionError reportUnsupportedEncoding() { - return new AssertionError(String.format("JVM doesn't support the %s encoding", TEXT_ENCODING)); - } -} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/aurora/blob/356eeac9/commons/src/main/java/org/apache/aurora/common/io/JsonCodec.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/io/JsonCodec.java b/commons/src/main/java/org/apache/aurora/common/io/JsonCodec.java deleted file mode 100644 index 1b955db..0000000 --- a/commons/src/main/java/org/apache/aurora/common/io/JsonCodec.java +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.common.io; - -import java.io.FilterOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.util.BitSet; - -import com.google.common.base.Preconditions; -import com.google.gson.ExclusionStrategy; -import com.google.gson.FieldAttributes; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; - -/** - * A {@code Codec} that can encode and decode objects to and from JSON using the GSON library - * (which in turn will use reflection). The codec uses the UTF-8 encoding. - * - * @author Attila Szegedi - */ -public class JsonCodec<T> implements Codec<T> { - - private static final String ENCODING = "utf-8"; - - private final Class<T> clazz; - private final Gson gson; - - /** - * Creates a new JSON codec instance for objects of the specified class. - * - * @param clazz the class of the objects the created codec is for. - * @return a newly constructed JSON codec instance for objects of the requested class. - */ - public static <T> JsonCodec<T> create(Class<T> clazz) { - return new JsonCodec<T>(clazz, DefaultGsonHolder.instance); - } - - /** - * Creates a new JSON codec instance for objects of the specified class and the specified Gson - * instance. You can use this method if you need to customize the behavior of the Gson - * serializer. - * - * @param clazz the class of the objects the created codec is for. - * @param gson the Gson instance to use for serialization/deserialization. - * @return a newly constructed JSON codec instance for objects of the requested class. - */ - public static <T> JsonCodec<T> create(Class<T> clazz, Gson gson) { - return new JsonCodec<T>(clazz, gson); - } - - private JsonCodec(Class<T> clazz, Gson gson) { - Preconditions.checkNotNull(clazz); - Preconditions.checkNotNull(gson); - this.clazz = clazz; - this.gson = gson; - } - - private static final class DefaultGsonHolder { - static final Gson instance = new Gson(); - } - - /** - * Returns a Gson exclusion strategy that excludes Thrift synthetic fields from JSON - * serialization. You can pass it to a {@link GsonBuilder} to construct a customized {@link Gson} - * instance to use with {@link JsonCodec#create(Class, Gson)}. - * - * @return a Gson exclusion strategy for thrift synthetic fields. - */ - public static ExclusionStrategy getThriftExclusionStrategy() { - return ThriftExclusionStrategy.instance; - } - - private static final class ThriftExclusionStrategy implements ExclusionStrategy { - static final ExclusionStrategy instance = new ThriftExclusionStrategy(); - - public boolean shouldSkipClass(Class<?> clazz) { - return false; - } - - public boolean shouldSkipField(FieldAttributes f) { - // Exclude Thrift synthetic fields - return f.getDeclaredClass() == BitSet.class && f.getName().equals("__isset_bit_vector"); - } - } - - @Override - public T deserialize(InputStream source) throws IOException { - return gson.fromJson(new InputStreamReader(source, ENCODING), clazz); - } - - @Override - public void serialize(T item, OutputStream sink) throws IOException { - final Writer w = new OutputStreamWriter(new UnflushableOutputStream(sink), ENCODING); - gson.toJson(item, clazz, w); - w.flush(); - } - - private static class UnflushableOutputStream extends FilterOutputStream { - UnflushableOutputStream(OutputStream out) { - super(out); - } - - @Override - public void flush() throws IOException { - // Intentionally do nothing - } - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/356eeac9/commons/src/main/java/org/apache/aurora/common/io/Streamer.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/io/Streamer.java b/commons/src/main/java/org/apache/aurora/common/io/Streamer.java deleted file mode 100644 index 9026760..0000000 --- a/commons/src/main/java/org/apache/aurora/common/io/Streamer.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.common.io; - -import com.google.common.base.Predicate; -import org.apache.aurora.common.base.Closure; - -/** - * Encapsulates iteration over a typed data stream that can be filtered. - * - * @author John Sirois - */ -public interface Streamer<T> { - - /** - * Processes a stream fully. This may cause a database query to be executed, a file to be read - * or even just call {@link Iterable#iterator()} depending on the implementation. Implementations - * guaranty that any resources allocated opening the stream will be closed whether or not process - * completes normally. - * - * @param work a closure over the work to be done for each item in the stream. - */ - void process(Closure<T> work); - - /** - * Returns a {@code Streamer} that will process the same stream as this streamer, but will stop - * processing when encountering the first item for which {@code cond} is true. - * - * @param cond a predicate that returns {@code false} as long as the stream should keep being - * processed. - * @return a streamer that will process items until the condition triggers. - */ - Streamer<T> endOn(Predicate<T> cond); - - /** - * Returns a {@code Streamer} that will process the same stream as this streamer, but with any - * items failing the filter to be omitted from processing. - * @param filter a predicate that returns {@code true} if an item in the stream should be - * processed - * @return a filtered streamer - */ - Streamer<T> filter(Predicate<T> filter); -} http://git-wip-us.apache.org/repos/asf/aurora/blob/356eeac9/commons/src/main/java/org/apache/aurora/common/logging/BufferedLog.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/logging/BufferedLog.java b/commons/src/main/java/org/apache/aurora/common/logging/BufferedLog.java deleted file mode 100644 index a83895d..0000000 --- a/commons/src/main/java/org/apache/aurora/common/logging/BufferedLog.java +++ /dev/null @@ -1,278 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.common.logging; - -import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import org.apache.aurora.common.quantity.Amount; -import org.apache.aurora.common.quantity.Time; -import org.apache.aurora.common.stats.StatImpl; -import org.apache.aurora.common.stats.Stats; - -import java.util.List; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.logging.Logger; - -/** - * Log that buffers requests before sending them to a wrapped log. - * - * @author William Farner - */ -public class BufferedLog<T, R> implements Log<T, Void> { - private static final Logger LOG = Logger.getLogger(BufferedLog.class.getName()); - - private static final ExecutorService DEFAULT_EXECUTOR_SERVICE = - Executors.newSingleThreadExecutor( - new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Log Pusher-%d").build()); - private static final int DEFAULT_MAX_BUFFER_SIZE = 100000; - - // TODO(William Farner): Change to use a ScheduledExecutorService instead of a timer. - private final TimerTask logPusher = new TimerTask() { - @Override public void run() { - flush(); - } - }; - - // Local buffer of log messages. - private final List<T> localBuffer = Lists.newLinkedList(); - - // The log that is being buffered. - private Log<T, R> bufferedLog; - - // Filter to determine when a log request should be retried. - private Predicate<R> retryFilter = null; - - // Maximum number of log entries that can be buffered before truncation (lost messages). - private int maxBufferSize = DEFAULT_MAX_BUFFER_SIZE; - - // Maximum buffer length before attempting to submit. - private int chunkLength; - - // Maximum time for a message to sit in the buffer before attempting to flush. - private Amount<Integer, Time> flushInterval; - - // Service to handle flushing the log. - private ExecutorService logSubmitService = DEFAULT_EXECUTOR_SERVICE; - - private BufferedLog() { - // Created through builder. - - Stats.export(new StatImpl<Integer>("scribe_buffer_size") { - public Integer read() { return getBacklog(); } - }); - } - - public static <T, R> Builder<T, R> builder() { - return new Builder<T, R>(); - } - - /** - * Starts the log submission service by scheduling a timer to periodically submit messages. - */ - private void start() { - long flushIntervalMillis = flushInterval.as(Time.MILLISECONDS); - - new Timer(true).scheduleAtFixedRate(logPusher, flushIntervalMillis, flushIntervalMillis); - } - - /** - * Gets the current number of messages in the local buffer. - * - * @return The number of backlogged messages. - */ - protected int getBacklog() { - synchronized (localBuffer) { - return localBuffer.size(); - } - } - - /** - * Stores a log entry, flushing immediately if the buffer length limit is exceeded. - * - * @param entry Entry to log. - */ - @Override - public Void log(T entry) { - synchronized (localBuffer) { - localBuffer.add(entry); - - if (localBuffer.size() >= chunkLength) { - logSubmitService.submit(logPusher); - } - } - - return null; - } - - @Override - public Void log(List<T> entries) { - for (T entry : entries) log(entry); - - return null; - } - - @Override - public void flush() { - List<T> buffer = copyBuffer(); - if (buffer.isEmpty()) return; - - R result = bufferedLog.log(buffer); - - // Restore the buffer if the write was not successful. - if (retryFilter != null && retryFilter.apply(result)) { - LOG.warning("Log request failed, restoring spooled messages."); - restoreToLocalBuffer(buffer); - } - } - - /** - * Creats a snapshot of the local buffer and clears the local buffer. - * - * @return A snapshot of the local buffer. - */ - private List<T> copyBuffer() { - synchronized (localBuffer) { - List<T> bufferCopy = ImmutableList.copyOf(localBuffer); - localBuffer.clear(); - return bufferCopy; - } - } - - /** - * Restores log entries back to the local buffer. This can be used to commit entries back to the - * buffer after a flush operation failed. - * - * @param buffer The log entries to restore. - */ - private void restoreToLocalBuffer(List<T> buffer) { - synchronized (localBuffer) { - int restoreRecords = Math.min(buffer.size(), maxBufferSize - localBuffer.size()); - - if (restoreRecords != buffer.size()) { - LOG.severe((buffer.size() - restoreRecords) + " log records truncated!"); - - if (restoreRecords == 0) return; - } - - localBuffer.addAll(0, buffer.subList(buffer.size() - restoreRecords, buffer.size())); - } - } - - /** - * Configures a BufferedLog object. - * - * @param <T> Log message type. - * @param <R> Log result type. - */ - public static class Builder<T, R> { - private final BufferedLog<T, R> instance; - - public Builder() { - instance = new BufferedLog<T, R>(); - } - - /** - * Specifies the log that should be buffered. - * - * @param bufferedLog Log to buffer requests to. - * @return A reference to the builder. - */ - public Builder<T, R> buffer(Log<T, R> bufferedLog) { - instance.bufferedLog = bufferedLog; - return this; - } - - /** - * Adds a custom retry filter that will be used to determine whether a log result {@code R} - * should be used to indicate that a log request should be retried. Log submit retry behavior - * is not defined when the filter throws uncaught exceptions. - * - * @param retryFilter Filter to determine whether to retry. - * @return A reference to the builder. - */ - public Builder<T, R> withRetryFilter(Predicate<R> retryFilter) { - instance.retryFilter = retryFilter; - return this; - } - - /** - * Specifies the maximum allowable buffer size, after which log records will be dropped to - * conserve memory. - * - * @param maxBufferSize Maximum buffer size. - * @return A reference to the builder. - */ - public Builder<T, R> withMaxBuffer(int maxBufferSize) { - instance.maxBufferSize = maxBufferSize; - return this; - } - - /** - * Specifies the desired number of log records to submit in each request. - * - * @param chunkLength Maximum number of records to accumulate before trying to submit. - * @return A reference to the builder. - */ - public Builder<T, R> withChunkLength(int chunkLength) { - instance.chunkLength = chunkLength; - return this; - } - - /** - * Specifies the maximum amount of time that a log entry may wait in the buffer before an - * attempt is made to flush the buffer. - * - * @param flushInterval Log flush interval. - * @return A reference to the builder. - */ - public Builder<T, R> withFlushInterval(Amount<Integer, Time> flushInterval) { - instance.flushInterval = flushInterval; - return this; - } - - /** - * Specifies the executor service to use for (synchronously or asynchronously) sending - * log entries. - * - * @param logSubmitService Log submit executor service. - * @return A reference to the builder. - */ - public Builder<T, R> withExecutorService(ExecutorService logSubmitService) { - instance.logSubmitService = logSubmitService; - return this; - } - - /** - * Creates the buffered log. - * - * @return The prepared buffered log. - */ - public BufferedLog<T, R> build() { - Preconditions.checkArgument(instance.chunkLength > 0); - Preconditions.checkArgument(instance.flushInterval.as(Time.MILLISECONDS) > 0); - Preconditions.checkNotNull(instance.logSubmitService); - Preconditions.checkArgument(instance.chunkLength <= instance.maxBufferSize); - - instance.start(); - - return instance; - } - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/356eeac9/commons/src/main/java/org/apache/aurora/common/logging/LogUtil.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/logging/LogUtil.java b/commons/src/main/java/org/apache/aurora/common/logging/LogUtil.java deleted file mode 100644 index 5f97cb2..0000000 --- a/commons/src/main/java/org/apache/aurora/common/logging/LogUtil.java +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.common.logging; - -import com.google.common.annotations.VisibleForTesting; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.SystemUtils; - -import java.io.File; -import java.util.logging.LogManager; -import java.util.logging.Logger; - -/** - * Logging utility functions. - * - * @author William Farner - */ -public class LogUtil { - - private static final Logger LOG = Logger.getLogger(LogUtil.class.getName()); - - private static final String LOG_MANAGER_FILE_PROP = "java.util.logging.FileHandler.pattern"; - - @VisibleForTesting - static final File DEFAULT_LOG_DIR = new File("/var/log"); - - /** - * Gets the log directory as configured with the log manager. This will attempt to expand any - * directory wildcards that are included in log file property. - * - * @return The configured log directory. - */ - public static File getLogManagerLogDir() { - return getLogManagerLogDir(LogManager.getLogManager().getProperty(LOG_MANAGER_FILE_PROP)); - } - - /** - * Gets the log directory as specified in a log file pattern. This will attempt to expand any - * directory wildcards that are included in log file property. - * - * @param logFilePattern The pattern to extract the log directory from. - * @return The configured log directory. - */ - public static File getLogManagerLogDir(String logFilePattern) { - if (StringUtils.isEmpty(logFilePattern)) { - LOG.warning("Could not find log dir in logging property " + LOG_MANAGER_FILE_PROP - + ", reading from " + DEFAULT_LOG_DIR); - return DEFAULT_LOG_DIR; - } - - String logDir = expandWildcard(logFilePattern, "%t", SystemUtils.JAVA_IO_TMPDIR); - logDir = expandWildcard(logDir, "%h", SystemUtils.USER_HOME); - File parent = new File(logDir).getParentFile(); - return parent == null ? new File(".") : parent; - } - - /** - * Expands a directory path wildcard within a file pattern string. - * Correctly handles cases where the replacement string does and does not contain a trailing - * slash. - * - * @param pattern File pattern string, which may or may not contain a wildcard. - * @param dirWildcard Wildcard string to expand. - * @param replacement Path component to expand wildcard to. - * @return {@code pattern} with all instances of {@code dirWildcard} replaced with - * {@code replacement}. - */ - private static String expandWildcard(String pattern, String dirWildcard, String replacement) { - String replace = dirWildcard; - if (replacement.charAt(replacement.length() - 1) == '/') { - replace += '/'; - } - return pattern.replaceAll(replace, replacement); - } - - private LogUtil() { - // Utility class. - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/356eeac9/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeHandler.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeHandler.java b/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeHandler.java deleted file mode 100644 index 8a9e18e..0000000 --- a/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeHandler.java +++ /dev/null @@ -1,196 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.common.logging.julbridge; - -import java.text.MessageFormat; -import java.util.MissingResourceException; -import java.util.logging.Formatter; -import java.util.logging.Handler; -import java.util.logging.LogRecord; - -import javax.annotation.Nullable; - -import org.apache.log4j.Level; -import org.apache.log4j.LogManager; -import org.apache.log4j.Logger; -import org.apache.log4j.spi.LocationInfo; -import org.apache.log4j.spi.LoggerRepository; -import org.apache.log4j.spi.LoggingEvent; -import org.apache.log4j.spi.ThrowableInformation; - -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * JUL Handler to convert JUL {@link LogRecord} messages into Log4j's {@link LoggingEvent} messages, - * and route them to a Log4J logger with the same name as the JUL logger. - */ -public class JULBridgeHandler extends Handler { - private static final String UNKNOWN_LOGGERNAME = "unknown"; - - /** - * Converts a JUL log record into a Log4J logging event. - * - * @param record the JUL log record to convert - * @param logger the Log4J logger to use for the logging event - * @param level the Log4J level to use for the logging event - * @param useExtendedLocationInfo if false, do no try to get source file and line informations - * @return a Log4J logging event - */ - static LoggingEvent toLoggingEvent(LogRecord record, Logger logger, Level level, - boolean useExtendedLocationInfo) { - - LocationInfo locationInfo = useExtendedLocationInfo - ? new LocationInfo(new Throwable(), record.getSourceClassName()) - : new LocationInfo("?", record.getSourceClassName(), record.getSourceMethodName(), "?"); - - // Getting thread name from thread id? complicated... - String threadName = String.valueOf(record.getThreadID()); - ThrowableInformation throwableInformation = record.getThrown() == null - ? null - : new ThrowableInformation(record.getThrown()); - - return new LoggingEvent( - record.getSourceClassName(), - logger, - record.getMillis(), - level, - formatMessage(record), - threadName, - throwableInformation, - null /* ndc */, - locationInfo, - null /* properties */); - } - - /** - * Formats a log record message in a way similar to {@link Formatter#formatMessage(LogRecord)}. - * - * If the record contains a resource bundle, a lookup is done to find a localized version. - * - * If the record contains parameters, the message is formatted using - * {@link MessageFormat#format(String, Object...)} - * - * @param record the log record used to format the message - * @return a formatted string - */ - static String formatMessage(LogRecord record) { - String message = record.getMessage(); - - // Look for a resource bundle - java.util.ResourceBundle catalog = record.getResourceBundle(); - if (catalog != null) { - try { - message = catalog.getString(record.getMessage()); - } catch (MissingResourceException e) { - // Not found? Fallback to original message string - message = record.getMessage(); - } - } - - Object parameters[] = record.getParameters(); - if (parameters == null || parameters.length == 0) { - // No parameters? just return the message string - return message; - } - - // Try formatting - try { - return MessageFormat.format(message, parameters); - } catch (IllegalArgumentException e) { - return message; - } - } - - private final LoggerRepository loggerRepository; - private final boolean useExtendedLocationInfo; - - /** - * Creates a new JUL handler. Equivalent to calling {@link #JULBridgeHandler(boolean)} passing - * <code>false</code> as argument. - */ - public JULBridgeHandler() { - this(LogManager.getLoggerRepository(), false); - } - - /** - * Creates a new JUL handler. - * Equivalent to calling {@link #JULBridgeHandler(LoggerRepository, boolean)} passing - * <code>LogManager.getLoggerRepository()</code> and <code>useExtendedLocationInfo</code> as - * arguments. - * - * @param useExtendedLocationInfo if true, try to add source filename and line info to log message - */ - public JULBridgeHandler(boolean useExtendedLocationInfo) { - this(LogManager.getLoggerRepository(), useExtendedLocationInfo); - } - - /** - * Creates a new JUL handler. - * - * @param loggerRepository Log4j logger repository where to get loggers from - * @param useExtendedLocationInfo if true, try to add source filename and line info to log message - * @throws NullPointerException if loggerRepository is null - */ - public JULBridgeHandler(LoggerRepository loggerRepository, boolean useExtendedLocationInfo) { - this.loggerRepository = checkNotNull(loggerRepository); - this.useExtendedLocationInfo = useExtendedLocationInfo; - } - - /** - * Gets a Log4J Logger with the same name as the logger name stored in the log record. - * - * @param record a JUL log record - * @return a Log4J logger with the same name, or name {@value #UNKNOWN_LOGGERNAME} if no name is - * present in the record. - */ - Logger getLogger(LogRecord record) { - String loggerName = record.getLoggerName(); - if (loggerName == null) { - loggerName = UNKNOWN_LOGGERNAME; - } - - return loggerRepository.getLogger(loggerName); - } - - /** - * Publishes the log record to a Log4J logger of the same name. - * - * Before formatting the message, level is converted and message is discarded if Log4j logger is - * not enabled for that level. - * - * @param record the record to publish - */ - @Override - public void publish(@Nullable LogRecord record) { - // Ignore silently null records - if (record == null) { - return; - } - - Logger log4jLogger = getLogger(record); - Level log4jLevel = JULBridgeLevelConverter.toLog4jLevel(record.getLevel()); - - if (log4jLogger.isEnabledFor(log4jLevel)) { - LoggingEvent event = toLoggingEvent(record, log4jLogger, log4jLevel, useExtendedLocationInfo); - - log4jLogger.callAppenders(event); - } - } - - @Override - public void flush() {} - - @Override - public void close() {} -} http://git-wip-us.apache.org/repos/asf/aurora/blob/356eeac9/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLevelConverter.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLevelConverter.java b/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLevelConverter.java deleted file mode 100644 index 14bbd4c..0000000 --- a/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLevelConverter.java +++ /dev/null @@ -1,99 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.common.logging.julbridge; - -import java.util.logging.Level; - -import static com.google.common.base.Preconditions.checkNotNull; - -/** - * An utility class to convert between JUL and Log4j Levels. Mapping is as follows: - * <ul> - * <li>FINEST <-> TRACE</li> - * <li>FINER -> DEBUG</li> - * <li>FINE <-> DEBUG</li> - * <li>INFO <-> INFO</li> - * <li>WARNING <-> WARN</li> - * <li>SEVERE <-> ERROR</li> - * <li>SEVERE <- FATAL</li> - * </ul> - * - * Unknowns levels are mapped to FINE/DEBUG - */ -public class JULBridgeLevelConverter { - - private JULBridgeLevelConverter() {} - - /** - * Converts a JUL level into a Log4j level. - * - * @param level the JUL level to convert - * @return a Log4j level - * @throws NullPointerException if level is null - */ - public static org.apache.log4j.Level toLog4jLevel(Level level) { - checkNotNull(level); - - if (level == Level.FINEST) { - return org.apache.log4j.Level.TRACE; - } else if (level == Level.FINER) { - return org.apache.log4j.Level.DEBUG; - } else if (level == Level.FINE) { - return org.apache.log4j.Level.DEBUG; - } else if (level == Level.INFO) { - return org.apache.log4j.Level.INFO; - } else if (level == Level.WARNING) { - return org.apache.log4j.Level.WARN; - } else if (level == Level.SEVERE) { - return org.apache.log4j.Level.ERROR; - } else if (level == Level.ALL) { - return org.apache.log4j.Level.ALL; - } else if (level == Level.OFF) { - return org.apache.log4j.Level.OFF; - } - - return org.apache.log4j.Level.DEBUG; - } - - /** - * Converts a Log4j level into a JUL level. - * - * @param level the Log4j level to convert - * @return a JUL level - * @throws NullPointerException if level is null - */ - public static Level fromLog4jLevel(org.apache.log4j.Level level) { - checkNotNull(level); - - if (level == org.apache.log4j.Level.TRACE) { - return Level.FINEST; - } else if (level == org.apache.log4j.Level.DEBUG) { - return Level.FINE; - } else if (level == org.apache.log4j.Level.INFO) { - return Level.INFO; - } else if (level == org.apache.log4j.Level.WARN) { - return Level.WARNING; - } else if (level == org.apache.log4j.Level.ERROR) { - return Level.SEVERE; - } else if (level == org.apache.log4j.Level.FATAL) { - return Level.SEVERE; - } else if (level == org.apache.log4j.Level.ALL) { - return Level.ALL; - } else if (level == org.apache.log4j.Level.OFF) { - return Level.OFF; - } - - return Level.FINE; - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/356eeac9/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLogManager.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLogManager.java b/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLogManager.java deleted file mode 100644 index adf1a83..0000000 --- a/commons/src/main/java/org/apache/aurora/common/logging/julbridge/JULBridgeLogManager.java +++ /dev/null @@ -1,87 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.common.logging.julbridge; - -import java.util.logging.LogManager; -import java.util.logging.Logger; - -import org.apache.log4j.spi.LoggerRepository; - -/** - * A JUL LogManager which takes over the logging configuration and redirects all messages to Log4j. - * - * The approach is inspired by the apache-jul-log4j-bridge project from Paul Smith - * (<[email protected]>) and available at <a - * href="http://people.apache.org/~psmith/logging.apache.org/sandbox/jul-log4j-bridge/"; /> - * - * During initialization, it resets configuration and adds a default handler to the root logger to - * perform the redirection. It also sets the root logger level to the Log4j repository threshold. - * This implies that Log4j is properly configured before this manager is taking over. - * - * To install this log manager, simply add the following property to the java command line: - * <code>-Djava.util.logging.manager=com.twitter.common.logging.julbridge.JULBridgeLogManager</code> - * - * It is possible to configure using extended location information (source filename and line info) - * by adding the following property to the java command line: - * <code>-Dcom.twitter.common.logging.julbridge.use-extended-location-info=true</code> - * - */ -public final class JULBridgeLogManager extends LogManager { - /** - * System property name to control if log messages sent from JUL to log4j should contain - * extended location information. - * - * Set @value to true to add source filename and line number to each message. - */ - public static final String USE_EXTENDED_LOCATION_INFO_PROPERTYNAME = - "com.twitter.common.logging.julbridge.use-extended-location-info"; - - /* - * LogManager requires a public no-arg constructor to be present so a new instance can be created - * when configured using the system property. A private constructor will throw an exception. - */ - public JULBridgeLogManager() {} - - @Override - public void readConfiguration() { - assimilate(org.apache.log4j.LogManager.getLoggerRepository()); - } - - /** - * Assimilates an existing JUL log manager. Equivalent to calling - * {@link #assimilate(LoggerRepository)} with <code>LogManager.getLoggerRepository</code>. - */ - public static void assimilate() { - assimilate(org.apache.log4j.LogManager.getLoggerRepository()); - } - - /** - * Assimilates an existing JUL log manager. - * - * It resets the manager configuration, and adds a bridge handler to the root logger. Messages are - * redirected to the specified Log4j logger repository. - * - * @param loggerRepository the Log4j logger repository to use to redirect messages - */ - public static void assimilate(LoggerRepository loggerRepository) { - LogManager.getLogManager().reset(); - - boolean withExtendedLocationInfos = - Boolean.getBoolean(USE_EXTENDED_LOCATION_INFO_PROPERTYNAME); - - Logger rootLogger = Logger.getLogger(""); - rootLogger.setLevel(JULBridgeLevelConverter.fromLog4jLevel(loggerRepository.getThreshold())); - rootLogger.addHandler(new JULBridgeHandler(loggerRepository, withExtendedLocationInfos)); - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/356eeac9/commons/src/main/java/org/apache/aurora/common/net/ProxyAuthorizer.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/net/ProxyAuthorizer.java b/commons/src/main/java/org/apache/aurora/common/net/ProxyAuthorizer.java deleted file mode 100644 index c5126dc..0000000 --- a/commons/src/main/java/org/apache/aurora/common/net/ProxyAuthorizer.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.common.net; - -import org.apache.commons.codec.binary.Base64; - -import java.net.HttpURLConnection; - -/** - * Authorizes http connection for use over the proxy it is built with - * - * @author William Farner - */ -public class ProxyAuthorizer { - private final ProxyConfig config; - - private ProxyAuthorizer(ProxyConfig config) { - this.config = config; - } - - public static ProxyAuthorizer adapt(ProxyConfig config) { - return new ProxyAuthorizer(config); - } - - public void authorize(HttpURLConnection httpCon) { - httpCon.setRequestProperty("Proxy-Authorization", "Basic " + - new String(Base64.encodeBase64(new String(config.getProxyUser() + ":" + - config.getProxyPassword()).getBytes())).trim()); - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/356eeac9/commons/src/main/java/org/apache/aurora/common/net/ProxyConfig.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/net/ProxyConfig.java b/commons/src/main/java/org/apache/aurora/common/net/ProxyConfig.java deleted file mode 100644 index fa474b5..0000000 --- a/commons/src/main/java/org/apache/aurora/common/net/ProxyConfig.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.common.net; - -import java.net.InetSocketAddress; -import javax.annotation.Nullable; - -/** - * Proxy configuration parameters: proxy address, username, and password. - * - * @author John Corwin - */ -public interface ProxyConfig { - public InetSocketAddress getProxyAddress(); - - public @Nullable String getProxyUser(); - - public @Nullable String getProxyPassword(); -} http://git-wip-us.apache.org/repos/asf/aurora/blob/356eeac9/commons/src/main/java/org/apache/aurora/common/net/UrlHelper.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/net/UrlHelper.java b/commons/src/main/java/org/apache/aurora/common/net/UrlHelper.java deleted file mode 100644 index 3f06d63..0000000 --- a/commons/src/main/java/org/apache/aurora/common/net/UrlHelper.java +++ /dev/null @@ -1,156 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.common.net; - -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; - -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.Arrays; -import java.util.List; -import java.util.logging.Logger; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -/** - * @author William Farner - */ -public class UrlHelper { - - private static final Logger LOG = Logger.getLogger(UrlHelper.class.getName()); - - /** - * Gets the domain from {@code url}. - * - * @param url A url. - * @return The domain portion of the URL, or {@code null} if the url is invalid. - */ - public static String getDomain(String url) { - try { - return getDomainChecked(url); - } catch (URISyntaxException e) { - LOG.finest("Malformed url: " + url); - return null; - } - } - - /** - * Gets the domain from {@code uri}, and throws an exception if it's not a valid uri. - * - * @param url A url. - * @throws URISyntaxException if url is not a valid {@code URI} - * @return The domain portion of the given url, or {@code null} if the host is undefined. - */ - public static String getDomainChecked(String url) throws URISyntaxException { - Preconditions.checkNotNull(url); - url = addProtocol(url); - return new URI(url).getHost(); - } - - /** - * Gets the path from {@code url}. - * - * @param url A url. - * @return The path portion of the URL, or {@code null} if the url is invalid. - */ - public static String getPath(String url) { - Preconditions.checkNotNull(url); - url = addProtocol(url); - try { - return new URI(url).getPath(); - } catch (URISyntaxException e) { - LOG.info("Malformed url: " + url); - return null; - } - } - - /** - * Strips URL parameters from a url. - * This will remove anything after and including a question mark in the URL. - * - * @param url The URL to strip parameters from. - * @return The original URL with parameters stripped, which will be the original URL if no - * parameters were found. - */ - public static String stripUrlParameters(String url) { - Preconditions.checkNotNull(url); - int paramStartIndex = url.indexOf("?"); - if (paramStartIndex == -1) { - return url; - } else { - return url.substring(0, paramStartIndex); - } - } - - /** - * Convenience method that calls #stripUrlParameters(String) for a URL. - * - * @param url The URL to strip parameters from. - * @return The original URL with parameters stripped, which will be the original URL if no - * parameters were found. - */ - public static String stripUrlParameters(URL url) { - return stripUrlParameters(url.toString()); - } - - private static final Pattern URL_PROTOCOL_REGEX = - Pattern.compile("^https?://", Pattern.CASE_INSENSITIVE); - - /** - * Checks whether a URL specifies its protocol, prepending http if it does not. - * - * @param url The URL to fix. - * @return The URL with the http protocol specified if no protocol was already specified. - */ - public static String addProtocol(String url) { - Preconditions.checkNotNull(url); - Matcher matcher = URL_PROTOCOL_REGEX.matcher(url); - if (!matcher.find()) { - url = "http://"; + url; - } - return url; - } - - /** - * Gets the domain levels for a host. - * For example, sub1.sub2.domain.co.uk would return - * [sub1.sub2.domain.co.uk, sub2.domain.co.uk, domain.co.uk, co.uk, uk]. - * - * - * @param host The host to peel subdomains off from. - * @return The domain levels in this host. - */ - public static List<String> getDomainLevels(String host) { - Preconditions.checkNotNull(host); - - // Automatically include www prefix if not present. - if (!host.startsWith("www")) { - host = "www." + host; - } - - Joiner joiner = Joiner.on("."); - List<String> domainParts = Lists.newLinkedList(Arrays.asList(host.split("\\."))); - List<String> levels = Lists.newLinkedList(); - - while (!domainParts.isEmpty()) { - levels.add(joiner.join(domainParts)); - domainParts.remove(0); - } - - return levels; - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/356eeac9/commons/src/main/java/org/apache/aurora/common/net/UrlResolver.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/net/UrlResolver.java b/commons/src/main/java/org/apache/aurora/common/net/UrlResolver.java deleted file mode 100644 index 96c5f07..0000000 --- a/commons/src/main/java/org/apache/aurora/common/net/UrlResolver.java +++ /dev/null @@ -1,446 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.common.net; - -import java.io.IOException; -import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.Semaphore; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; -import java.util.logging.Logger; - -import javax.annotation.Nullable; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Functions; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; -import com.google.common.collect.Lists; -import com.google.common.util.concurrent.ListenableFutureTask; -import com.google.common.util.concurrent.ThreadFactoryBuilder; - -import org.apache.aurora.common.base.ExceptionalFunction; -import org.apache.aurora.common.net.UrlResolver.ResolvedUrl.EndState; -import org.apache.aurora.common.quantity.Amount; -import org.apache.aurora.common.quantity.Time; -import org.apache.aurora.common.stats.PrintableHistogram; -import org.apache.aurora.common.util.BackoffStrategy; -import org.apache.aurora.common.util.Clock; -import org.apache.aurora.common.util.TruncatedBinaryBackoff; -import org.apache.aurora.common.util.caching.Cache; -import org.apache.aurora.common.util.caching.LRUCache; - -/** - * Class to aid in resolving URLs by following redirects, which can optionally be performed - * asynchronously using a thread pool. - * - * @author William Farner - */ -public class UrlResolver { - private static final Logger LOG = Logger.getLogger(UrlResolver.class.getName()); - - private static final String TWITTER_UA = "Twitterbot/0.1"; - private static final UrlResolverUtil URL_RESOLVER = - new UrlResolverUtil(Functions.constant(TWITTER_UA)); - - private static final ExceptionalFunction<String, String, IOException> RESOLVER = - new ExceptionalFunction<String, String, IOException>() { - @Override public String apply(String url) throws IOException { - return URL_RESOLVER.getEffectiveUrl(url, null); - } - }; - - private static ExceptionalFunction<String, String, IOException> - getUrlResolver(final @Nullable ProxyConfig proxyConfig) { - if (proxyConfig != null) { - return new ExceptionalFunction<String, String, IOException>() { - @Override public String apply(String url) throws IOException { - return URL_RESOLVER.getEffectiveUrl(url, proxyConfig); - } - }; - } else { - return RESOLVER; - } - } - - private final ExceptionalFunction<String, String, IOException> resolver; - private final int maxRedirects; - - // Tracks the number of active tasks (threads in use). - private final Semaphore poolEntrySemaphore; - private final Integer threadPoolSize; - - // Helps with signaling the handler. - private final Executor handlerExecutor; - - // Manages the thread pool and task execution. - private ExecutorService executor; - - // Cache to store resolved URLs. - private final Cache<String, String> urlCache = LRUCache.<String, String>builder() - .maxSize(10000) - .makeSynchronized(true) - .build(); - - // Variables to track connection/request stats. - private AtomicInteger requestCount = new AtomicInteger(0); - private AtomicInteger cacheHits = new AtomicInteger(0); - private AtomicInteger failureCount = new AtomicInteger(0); - // Tracks the time (in milliseconds) required to resolve URLs. - private final PrintableHistogram urlResolutionTimesMs = new PrintableHistogram( - 1, 5, 10, 25, 50, 75, 100, 150, 200, 250, 300, 500, 750, 1000, 1500, 2000); - - private final Clock clock; - private final BackoffStrategy backoffStrategy; - - @VisibleForTesting - UrlResolver(Clock clock, BackoffStrategy backoffStrategy, - ExceptionalFunction<String, String, IOException> resolver, int maxRedirects) { - this(clock, backoffStrategy, resolver, maxRedirects, null); - } - - /** - * Creates a new asynchronous URL resolver. A thread pool will be used to resolve URLs, and - * resolved URLs will be announced via {@code handler}. - * - * @param maxRedirects The maximum number of HTTP redirects to follow. - * @param threadPoolSize The number of threads to use for resolving URLs. - * @param proxyConfig The proxy settings with which to make the HTTP request, or null for the - * default configured proxy. - */ - public UrlResolver(int maxRedirects, int threadPoolSize, @Nullable ProxyConfig proxyConfig) { - this(Clock.SYSTEM_CLOCK, - new TruncatedBinaryBackoff(Amount.of(100L, Time.MILLISECONDS), Amount.of(1L, Time.SECONDS)), - getUrlResolver(proxyConfig), maxRedirects, threadPoolSize); - } - - public UrlResolver(int maxRedirects, int threadPoolSize) { - this(maxRedirects, threadPoolSize, null); - } - - private UrlResolver(Clock clock, BackoffStrategy backoffStrategy, - ExceptionalFunction<String, String, IOException> resolver, int maxRedirects, - @Nullable Integer threadPoolSize) { - this.clock = clock; - this.backoffStrategy = backoffStrategy; - this.resolver = resolver; - this.maxRedirects = maxRedirects; - - if (threadPoolSize != null) { - this.threadPoolSize = threadPoolSize; - Preconditions.checkState(threadPoolSize > 0); - poolEntrySemaphore = new Semaphore(threadPoolSize); - - // Start up the thread pool. - reset(); - - // Executor to send notifications back to the handler. This also needs to be - // a daemon thread. - handlerExecutor = - Executors.newSingleThreadExecutor(new ThreadFactoryBuilder().setDaemon(true).build()); - } else { - this.threadPoolSize = null; - poolEntrySemaphore = null; - handlerExecutor = null; - } - } - - public Future<ResolvedUrl> resolveUrlAsync(final String url, final ResolvedUrlHandler handler) { - Preconditions.checkNotNull( - "Asynchronous URL resolution cannot be performed without a valid handler.", handler); - - try { - poolEntrySemaphore.acquire(); - } catch (InterruptedException e) { - LOG.log(Level.SEVERE, "Interrupted while waiting for thread to resolve URL: " + url, e); - return null; - } - final ListenableFutureTask<ResolvedUrl> future = - ListenableFutureTask.create( - new Callable<ResolvedUrl>() { - @Override public ResolvedUrl call() { - return resolveUrl(url); - } - }); - - future.addListener(new Runnable() { - @Override public void run() { - try { - handler.resolved(future); - } finally { - poolEntrySemaphore.release(); - } - } - }, handlerExecutor); - - executor.execute(future); - return future; - } - - private void logThreadpoolInfo() { - LOG.info("Shutting down thread pool, available permits: " - + poolEntrySemaphore.availablePermits()); - LOG.info("Queued threads? " + poolEntrySemaphore.hasQueuedThreads()); - LOG.info("Queue length: " + poolEntrySemaphore.getQueueLength()); - } - - public void reset() { - Preconditions.checkState(threadPoolSize != null); - if (executor != null) { - Preconditions.checkState(executor.isShutdown(), - "The thread pool must be shut down before resetting."); - Preconditions.checkState(executor.isTerminated(), "There may still be pending async tasks."); - } - - // Create a thread pool with daemon threads, so that they may be terminated when no - // application threads are running. - executor = Executors.newFixedThreadPool(threadPoolSize, - new ThreadFactoryBuilder().setDaemon(true).setNameFormat("UrlResolver[%d]").build()); - } - - /** - * Terminates the thread pool, waiting at most {@code waitSeconds} for active threads to complete. - * After this method is called, no more URLs may be submitted for resolution. - * - * @param waitSeconds The number of seconds to wait for active threads to complete. - */ - public void clearAsyncTasks(int waitSeconds) { - Preconditions.checkState(threadPoolSize != null, - "finish() should not be called on a synchronous URL resolver."); - - logThreadpoolInfo(); - executor.shutdown(); // Disable new tasks from being submitted. - try { - // Wait a while for existing tasks to terminate - if (!executor.awaitTermination(waitSeconds, TimeUnit.SECONDS)) { - LOG.info("Pool did not terminate, forcing shutdown."); - logThreadpoolInfo(); - List<Runnable> remaining = executor.shutdownNow(); - LOG.info("Tasks still running: " + remaining); - // Wait a while for tasks to respond to being cancelled - if (!executor.awaitTermination(waitSeconds, TimeUnit.SECONDS)) { - LOG.warning("Pool did not terminate."); - logThreadpoolInfo(); - } - } - } catch (InterruptedException e) { - LOG.log(Level.WARNING, "Interrupted while waiting for threadpool to finish.", e); - // (Re-)Cancel if current thread also interrupted - executor.shutdownNow(); - // Preserve interrupt status - Thread.currentThread().interrupt(); - } - } - - /** - * Resolves a URL synchronously. - * - * @param url The URL to resolve. - * @return The resolved URL. - */ - public ResolvedUrl resolveUrl(String url) { - ResolvedUrl resolvedUrl = new ResolvedUrl(); - resolvedUrl.setStartUrl(url); - - String cached = urlCache.get(url); - if (cached != null) { - cacheHits.incrementAndGet(); - resolvedUrl.setNextResolve(cached); - resolvedUrl.setEndState(EndState.CACHED); - return resolvedUrl; - } - - String currentUrl = url; - long backoffMs = 0L; - String next = null; - for (int i = 0; i < maxRedirects; i++) { - try { - next = resolveOnce(currentUrl); - - // If there was a 4xx or a 5xx, we''ll get a null back, so we pretend like we never advanced - // to allow for a retry within the redirect limit. - // TODO(John Sirois): we really need access to the return code here to do the right thing; ie: - // retry for internal server errors but probably not for unauthorized - if (next == null) { - if (i < maxRedirects - 1) { // don't wait if we're about to exit the loop - backoffMs = backoffStrategy.calculateBackoffMs(backoffMs); - try { - clock.waitFor(backoffMs); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException( - "Interrupted waiting to retry a failed resolution for: " + currentUrl, e); - } - } - continue; - } - - backoffMs = 0L; - if (next.equals(currentUrl)) { - // We've reached the end of the redirect chain. - resolvedUrl.setEndState(EndState.REACHED_LANDING); - urlCache.put(url, currentUrl); - for (String intermediateUrl : resolvedUrl.getIntermediateUrls()) { - urlCache.put(intermediateUrl, currentUrl); - } - return resolvedUrl; - } else if (!url.equals(next)) { - resolvedUrl.setNextResolve(next); - } - currentUrl = next; - } catch (IOException e) { - LOG.log(Level.INFO, "Failed to resolve url: " + url, e); - resolvedUrl.setEndState(EndState.ERROR); - return resolvedUrl; - } - } - - resolvedUrl.setEndState(next == null || url.equals(currentUrl) ? EndState.ERROR - : EndState.REDIRECT_LIMIT); - return resolvedUrl; - } - - /** - * Resolves a url, following at most one redirect. Thread-safe. - * - * @param url The URL to resolve. - * @return The result of following the URL through at most one redirect or null if the url could - * not be followed - * @throws IOException If an error occurs while resolving the URL. - */ - private String resolveOnce(String url) throws IOException { - requestCount.incrementAndGet(); - - String resolvedUrl = urlCache.get(url); - if (resolvedUrl != null) { - cacheHits.incrementAndGet(); - return resolvedUrl; - } - - try { - long startTimeMs = System.currentTimeMillis(); - resolvedUrl = resolver.apply(url); - if (resolvedUrl == null) { - return null; - } - - urlCache.put(url, resolvedUrl); - - synchronized (urlResolutionTimesMs) { - urlResolutionTimesMs.addValue(System.currentTimeMillis() - startTimeMs); - } - return resolvedUrl; - } catch (IOException e) { - failureCount.incrementAndGet(); - throw e; - } - } - - @Override - public String toString() { - return String.format("Cache: %s\nFailed requests: %d,\nResolution Times: %s", - urlCache, failureCount.get(), - urlResolutionTimesMs.toString()); - } - - /** - * Class to wrap the result of a URL resolution. - */ - public static class ResolvedUrl { - public enum EndState { - REACHED_LANDING, - ERROR, - CACHED, - REDIRECT_LIMIT - } - - private String startUrl; - private final List<String> resolveChain; - private EndState endState; - - public ResolvedUrl() { - resolveChain = Lists.newArrayList(); - } - - @VisibleForTesting - public ResolvedUrl(EndState endState, String startUrl, String... resolveChain) { - this.endState = endState; - this.startUrl = startUrl; - this.resolveChain = Lists.newArrayList(resolveChain); - } - - public String getStartUrl() { - return startUrl; - } - - void setStartUrl(String startUrl) { - this.startUrl = startUrl; - } - - /** - * Returns the last URL resolved following a redirect chain, or null if the startUrl is a - * landing URL. - */ - public String getEndUrl() { - return resolveChain.isEmpty() ? null : Iterables.getLast(resolveChain); - } - - void setNextResolve(String endUrl) { - this.resolveChain.add(endUrl); - } - - /** - * Returns any immediate URLs encountered on the resolution chain. If the startUrl redirects - * directly to the endUrl or they are the same the imtermediate URLs will be empty. - */ - public Iterable<String> getIntermediateUrls() { - return resolveChain.size() <= 1 ? ImmutableList.<String>of() - : resolveChain.subList(0, resolveChain.size() - 1); - } - - public EndState getEndState() { - return endState; - } - - void setEndState(EndState endState) { - this.endState = endState; - } - - public String toString() { - return String.format("%s -> %s [%s, %d redirects]", - startUrl, Joiner.on(" -> ").join(resolveChain), endState, resolveChain.size()); - } - } - - /** - * Interface to use for notifying the caller of resolved URLs. - */ - public interface ResolvedUrlHandler { - /** - * Signals that a URL has been resolved to its target. The implementation of this method must - * be thread safe. - * - * @param future The future that has finished resolving a URL. - */ - public void resolved(Future<ResolvedUrl> future); - } -} http://git-wip-us.apache.org/repos/asf/aurora/blob/356eeac9/commons/src/main/java/org/apache/aurora/common/net/UrlResolverUtil.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/org/apache/aurora/common/net/UrlResolverUtil.java b/commons/src/main/java/org/apache/aurora/common/net/UrlResolverUtil.java deleted file mode 100644 index 4b95bb7..0000000 --- a/commons/src/main/java/org/apache/aurora/common/net/UrlResolverUtil.java +++ /dev/null @@ -1,148 +0,0 @@ -/** - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.aurora.common.net; - -import com.google.common.base.Function; -import com.google.common.base.Functions; -import com.google.common.base.Preconditions; -import org.apache.aurora.common.base.MorePreconditions; -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.Proxy; -import java.net.Proxy.Type; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.Map; -import java.util.logging.Logger; -import javax.annotation.Nullable; - -/** - * A utility that can resolve HTTP urls. - * - * @author John Sirois - */ -class UrlResolverUtil { - - private static final Logger LOG = Logger.getLogger(UrlResolverUtil.class.getName()); - - // Default user-agent string to user for HTTP requests. - private static final String DEFAULT_USER_AGENT = "Lynxy/6.6.6dev.8 libwww-FM/3.14159FM"; - - private static Map<String, String> checkNotBlank(Map<String, String> hostToUserAgent) { - Preconditions.checkNotNull(hostToUserAgent); - MorePreconditions.checkNotBlank(hostToUserAgent.entrySet()); - return hostToUserAgent; - } - - private final Function<? super URL, String> urlToUserAgent; - - UrlResolverUtil(Map<String, String> hostToUserAgent) { - this(Functions.compose(Functions.forMap(checkNotBlank(hostToUserAgent), DEFAULT_USER_AGENT), - new Function<URL, String>() { - @Override public String apply(URL url) { - return url.getHost(); - } - })); - } - - UrlResolverUtil(Function<? super URL, String> urlToUserAgent) { - this.urlToUserAgent = Preconditions.checkNotNull(urlToUserAgent); - } - - /** - * Returns the URL that {@code url} lands on, which will be the result of a 3xx redirect, - * or {@code url} if the url does not redirect using an HTTP 3xx response code. If there is a - * non-2xx or 3xx HTTP response code null is returned. - * - * @param url The URL to follow. - * @return The redirected URL, or {@code url} if {@code url} returns a 2XX response, otherwise - * null - * @throws java.io.IOException If an error occurs while trying to follow the url. - */ - String getEffectiveUrl(String url, @Nullable ProxyConfig proxyConfig) throws IOException { - Preconditions.checkNotNull(url); - // Don't follow https. - if (url.startsWith("https://";)) { - url = url.replace("https://";, "http://";); - } else if (!url.startsWith("http://";)) { - url = "http://"; + url; - } - - URL urlObj = new URL(url); - - HttpURLConnection con; - if (proxyConfig != null) { - Proxy proxy = new Proxy(Type.HTTP, proxyConfig.getProxyAddress()); - con = (HttpURLConnection) urlObj.openConnection(proxy); - ProxyAuthorizer.adapt(proxyConfig).authorize(con); - } else { - con = (HttpURLConnection) urlObj.openConnection(); - } - try { - - // TODO(John Sirois): several commonly tweeted hosts 406 or 400 on HEADs and only work with GETs - // fix the call chain to be able to specify retry-with-GET - con.setRequestMethod("HEAD"); - - con.setUseCaches(true); - con.setConnectTimeout(5000); - con.setReadTimeout(5000); - con.setInstanceFollowRedirects(false); - - // I hate to have to do this, but some URL shorteners don't respond otherwise. - con.setRequestProperty("User-Agent", urlToUserAgent.apply(urlObj)); - try { - con.connect(); - } catch (StringIndexOutOfBoundsException e) { - LOG.info("Got StringIndexOutOfBoundsException when fetching headers for " + url); - return null; - } - - int responseCode = con.getResponseCode(); - switch (responseCode / 100) { - case 2: - return url; - case 3: - String location = con.getHeaderField("Location"); - if (location == null) { - if (responseCode != 304 /* not modified */) { - LOG.info( - String.format("[%d] Location header was null for URL: %s", responseCode, url)); - } - return url; - } - - // HTTP 1.1 spec says this should be an absolute URI, but i see lots of instances where it - // is relative, so we need to check. - try { - String domain = UrlHelper.getDomainChecked(location); - if (domain == null || domain.isEmpty()) { - // This is a relative URI. - location = "http://"; + UrlHelper.getDomain(url) + location; - } - } catch (URISyntaxException e) { - LOG.info("location contained an invalid URI: " + location); - } - - return location; - default: - LOG.info("Failed to resolve url: " + url + " with: " - + responseCode + " -> " + con.getResponseMessage()); - return null; - } - } finally { - con.disconnect(); - } - } -}
