http://git-wip-us.apache.org/repos/asf/aurora/blob/86a547b9/commons/src/main/java/com/twitter/common/io/CompatibilityCodec.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/com/twitter/common/io/CompatibilityCodec.java b/commons/src/main/java/com/twitter/common/io/CompatibilityCodec.java new file mode 100644 index 0000000..1274f94 --- /dev/null +++ b/commons/src/main/java/com/twitter/common/io/CompatibilityCodec.java @@ -0,0 +1,97 @@ +// ================================================================================================= +// Copyright 2011 Twitter, Inc. +// ------------------------------------------------------------------------------------------------- +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this work except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file, or at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ================================================================================================= +package com.twitter.common.io; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PushbackInputStream; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; + +/** + * A codec that composes two codecs: a primary and a compatibility codec. It always serializes with + * the primary codec, but can make a decision on deserialization based on the first few bytes of the + * serialized format whether to use the compatibility codec. This allows for easier transition + * between storage formats as the codec remains able to read the old serialized format. + * + * @author Attila Szegedi + * + * @param <T> the type of objects this codec is for. + */ +public class CompatibilityCodec<T> implements Codec<T> { + private final Codec<T> primaryCodec; + private final Codec<T> secondaryCodec; + private final int prefixLength; + private final Predicate<byte[]> discriminator; + + private CompatibilityCodec(Codec<T> primaryCodec, Codec<T> secondaryCodec, int prefixLength, + Predicate<byte[]> discriminator) { + Preconditions.checkNotNull(primaryCodec); + Preconditions.checkNotNull(secondaryCodec); + this.primaryCodec = primaryCodec; + this.secondaryCodec = secondaryCodec; + this.prefixLength = prefixLength; + this.discriminator = discriminator; + } + + /** + * Creates a new compatibility codec instance. + * + * @param primaryCodec the codec used to serialize objects, as well as deserialize them when the + * first byte of the serialized format matches the discriminator. + * @param secondaryCodec the codec used to deserialize objects when the first byte of the + * serialized format does not match the discriminator. + * @param prefixLength the length, in bytes, of the prefix of the message that is inspected for + * determining the format. + * @param discriminator a predicate that will receive an array of at most prefixLength bytes + * (it can receive less if the serialized format is shorter) and has to return true + * if the primary codec should be used for deserialization, otherwise false. + */ + public static <T> CompatibilityCodec<T> create(Codec<T> primaryCodec, Codec<T> secondaryCodec, + int prefixLength, Predicate<byte[]> discriminator) { + return new CompatibilityCodec<T>(primaryCodec, secondaryCodec, prefixLength, discriminator); + } + + @Override + public T deserialize(InputStream source) throws IOException { + final PushbackInputStream in = new PushbackInputStream(source, prefixLength); + final byte[] prefix = readAtMostPrefix(in); + in.unread(prefix); + return (discriminator.apply(prefix) ? primaryCodec : secondaryCodec).deserialize(in); + } + + private byte[] readAtMostPrefix(InputStream in) throws IOException { + final byte[] prefix = new byte[prefixLength]; + int read = 0; + do { + final int readNow = in.read(prefix, read, prefixLength - read); + if (readNow == -1) { + byte[] newprefix = new byte[read]; + System.arraycopy(prefix, 0, newprefix, 0, read); + return newprefix; + } + read += readNow; + } while (read < prefixLength); + return prefix; + } + + @Override + public void serialize(T item, OutputStream sink) throws IOException { + primaryCodec.serialize(item, sink); + } +}
http://git-wip-us.apache.org/repos/asf/aurora/blob/86a547b9/commons/src/main/java/com/twitter/common/io/FileUtils.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/com/twitter/common/io/FileUtils.java b/commons/src/main/java/com/twitter/common/io/FileUtils.java new file mode 100644 index 0000000..454866a --- /dev/null +++ b/commons/src/main/java/com/twitter/common/io/FileUtils.java @@ -0,0 +1,196 @@ +// ================================================================================================= +// Copyright 2011 Twitter, Inc. +// ------------------------------------------------------------------------------------------------- +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this work except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file, or at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ================================================================================================= + +package com.twitter.common.io; + +import java.io.File; +import java.io.IOException; +import java.util.UUID; + +import com.google.common.base.Preconditions; + +import org.apache.commons.lang.SystemUtils; + +import com.twitter.common.base.ExceptionalClosure; +import com.twitter.common.base.ExceptionalFunction; + +/** + * Utility methods for working with files and directories. + * + * @author John Sirois + */ +public final class FileUtils { + + /** + * A utility for creating and working with temporary files and directories. + */ + public static class Temporary { + private static final int MAX_TMP_DIR_TRIES = 5; + + private final File basedir; + + /** + * Creates a new temporary utility that creates files and directories rooted at {@code basedir}. + * + * @param basedir The base directory to generate temporary files and directories in. + */ + public Temporary(File basedir) { + Preconditions.checkNotNull(basedir); + this.basedir = basedir; + } + + /** + * Returns a new empty temporary directory. + * + * @return a file representing the newly created directory. + * @throws IllegalStateException if a new temporary directory could not be created + */ + public File createDir() { + File tempDir; + int tries = 0; + do { + // For sanity sake, die eventually if we keep failing to pick a new unique directory name. + if (++tries > MAX_TMP_DIR_TRIES) { + throw new IllegalStateException("Failed to create a new temp directory in " + + MAX_TMP_DIR_TRIES + " attempts, giving up"); + } + tempDir = new File(basedir, UUID.randomUUID().toString()); + } while (!tempDir.mkdir()); + return tempDir; + } + + /** + * Creates a new empty temporary file. + * + * @return a new empty temporary file + * @throws IOException if there was a problem creating a new temporary file + */ + public File createFile() throws IOException { + return createFile(".tempfile"); + } + + /** + * Creates a new empty temporary file with the given filename {@code suffix}. + * + * @param suffix The suffix for the temporary file name + * @return a new empty temporary file + * @throws IOException if there was a problem creating a new temporary file + */ + public File createFile(String suffix) throws IOException { + return File.createTempFile(FileUtils.class.getName(), suffix, basedir); + } + + /** + * Creates a new temporary directory and executes the unit of {@code work} against it ensuring + * the directory and its contents are removed after the work completes normally or abnormally. + * + * @param work The unit of work to execute against the new temporary directory. + * @param <E> The type of exception this unit of work can throw. + * @throws E bubbled transparently when the unit of work throws + */ + public <E extends Exception> void doWithDir(final ExceptionalClosure<File, E> work) + throws E { + Preconditions.checkNotNull(work); + doWithDir(new ExceptionalFunction<File, Void, E>() { + @Override public Void apply(File dir) throws E { + work.execute(dir); + return null; + } + }); + } + + /** + * Creates a new temporary directory and executes the unit of {@code work} against it ensuring + * the directory and its contents are removed after the work completes normally or abnormally. + * + * @param work The unit of work to execute against the new temporary directory. + * @param <T> The type of result this unit of work produces. + * @param <E> The type of exception this unit of work can throw. + * @return the result when the unit of work completes successfully + * @throws E bubbled transparently when the unit of work throws + */ + public <T, E extends Exception> T doWithDir(ExceptionalFunction<File, T, E> work) + throws E { + Preconditions.checkNotNull(work); + return doWithTemp(createDir(), work); + } + + /** + * Creates a new temporary file and executes the unit of {@code work} against it ensuring + * the file is removed after the work completes normally or abnormally. + * + * @param work The unit of work to execute against the new temporary file. + * @param <E> The type of exception this unit of work can throw. + * @throws E bubbled transparently when the unit of work throws + * @throws IOException if there was a problem creating a new temporary file + */ + public <E extends Exception> void doWithFile(final ExceptionalClosure<File, E> work) + throws E, IOException { + Preconditions.checkNotNull(work); + doWithFile(new ExceptionalFunction<File, Void, E>() { + @Override public Void apply(File dir) throws E { + work.execute(dir); + return null; + } + }); + } + + /** + * Creates a new temporary file and executes the unit of {@code work} against it ensuring + * the file is removed after the work completes normally or abnormally. + * + * @param work The unit of work to execute against the new temporary file. + * @param <T> The type of result this unit of work produces. + * @param <E> The type of exception this unit of work can throw. + * @return the result when the unit of work completes successfully + * @throws E bubbled transparently when the unit of work throws + * @throws IOException if there was a problem creating a new temporary file + */ + public <T, E extends Exception> T doWithFile(ExceptionalFunction<File, T, E> work) + throws E, IOException { + Preconditions.checkNotNull(work); + return doWithTemp(createFile(), work); + } + + private static <T, E extends Exception> T doWithTemp(File file, + ExceptionalFunction<File, T, E> work) throws E { + try { + return work.apply(file); + } finally { + org.apache.commons.io.FileUtils.deleteQuietly(file); + } + } + } + + /** + * A temporary based at the default system temporary directory. + */ + public static final Temporary SYSTEM_TMP = new Temporary(SystemUtils.getJavaIoTmpDir()); + + /** + * Returns a new empty temporary directory. + * + * @return a file representing the newly created directory. + * @throws IllegalStateException if a new temporary directory could not be created + */ + public static File createTempDir() { + return SYSTEM_TMP.createDir(); + } + + private FileUtils() { + // utility + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86a547b9/commons/src/main/java/com/twitter/common/io/JsonCodec.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/com/twitter/common/io/JsonCodec.java b/commons/src/main/java/com/twitter/common/io/JsonCodec.java new file mode 100644 index 0000000..25bc6ce --- /dev/null +++ b/commons/src/main/java/com/twitter/common/io/JsonCodec.java @@ -0,0 +1,127 @@ +// ================================================================================================= +// Copyright 2011 Twitter, Inc. +// ------------------------------------------------------------------------------------------------- +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this work except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file, or at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ================================================================================================= + +package com.twitter.common.io; + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.util.BitSet; + +import com.google.common.base.Preconditions; +import com.google.gson.ExclusionStrategy; +import com.google.gson.FieldAttributes; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + +/** + * A {@code Codec} that can encode and decode objects to and from JSON using the GSON library + * (which in turn will use reflection). The codec uses the UTF-8 encoding. + * + * @author Attila Szegedi + */ +public class JsonCodec<T> implements Codec<T> { + + private static final String ENCODING = "utf-8"; + + private final Class<T> clazz; + private final Gson gson; + + /** + * Creates a new JSON codec instance for objects of the specified class. + * + * @param clazz the class of the objects the created codec is for. + * @return a newly constructed JSON codec instance for objects of the requested class. + */ + public static <T> JsonCodec<T> create(Class<T> clazz) { + return new JsonCodec<T>(clazz, DefaultGsonHolder.instance); + } + + /** + * Creates a new JSON codec instance for objects of the specified class and the specified Gson + * instance. You can use this method if you need to customize the behavior of the Gson + * serializer. + * + * @param clazz the class of the objects the created codec is for. + * @param gson the Gson instance to use for serialization/deserialization. + * @return a newly constructed JSON codec instance for objects of the requested class. + */ + public static <T> JsonCodec<T> create(Class<T> clazz, Gson gson) { + return new JsonCodec<T>(clazz, gson); + } + + private JsonCodec(Class<T> clazz, Gson gson) { + Preconditions.checkNotNull(clazz); + Preconditions.checkNotNull(gson); + this.clazz = clazz; + this.gson = gson; + } + + private static final class DefaultGsonHolder { + static final Gson instance = new Gson(); + } + + /** + * Returns a Gson exclusion strategy that excludes Thrift synthetic fields from JSON + * serialization. You can pass it to a {@link GsonBuilder} to construct a customized {@link Gson} + * instance to use with {@link JsonCodec#create(Class, Gson)}. + * + * @return a Gson exclusion strategy for thrift synthetic fields. + */ + public static ExclusionStrategy getThriftExclusionStrategy() { + return ThriftExclusionStrategy.instance; + } + + private static final class ThriftExclusionStrategy implements ExclusionStrategy { + static final ExclusionStrategy instance = new ThriftExclusionStrategy(); + + public boolean shouldSkipClass(Class<?> clazz) { + return false; + } + + public boolean shouldSkipField(FieldAttributes f) { + // Exclude Thrift synthetic fields + return f.getDeclaredClass() == BitSet.class && f.getName().equals("__isset_bit_vector"); + } + } + + @Override + public T deserialize(InputStream source) throws IOException { + return gson.fromJson(new InputStreamReader(source, ENCODING), clazz); + } + + @Override + public void serialize(T item, OutputStream sink) throws IOException { + final Writer w = new OutputStreamWriter(new UnflushableOutputStream(sink), ENCODING); + gson.toJson(item, clazz, w); + w.flush(); + } + + private static class UnflushableOutputStream extends FilterOutputStream { + UnflushableOutputStream(OutputStream out) { + super(out); + } + + @Override + public void flush() throws IOException { + // Intentionally do nothing + } + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86a547b9/commons/src/main/java/com/twitter/common/io/Streamer.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/com/twitter/common/io/Streamer.java b/commons/src/main/java/com/twitter/common/io/Streamer.java new file mode 100644 index 0000000..27ae968 --- /dev/null +++ b/commons/src/main/java/com/twitter/common/io/Streamer.java @@ -0,0 +1,57 @@ +// ================================================================================================= +// Copyright 2011 Twitter, Inc. +// ------------------------------------------------------------------------------------------------- +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this work except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file, or at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ================================================================================================= + +package com.twitter.common.io; + +import com.google.common.base.Predicate; +import com.twitter.common.base.Closure; + +/** + * Encapsulates iteration over a typed data stream that can be filtered. + * + * @author John Sirois + */ +public interface Streamer<T> { + + /** + * Processes a stream fully. This may cause a database query to be executed, a file to be read + * or even just call {@link Iterable#iterator()} depending on the implementation. Implementations + * guaranty that any resources allocated opening the stream will be closed whether or not process + * completes normally. + * + * @param work a closure over the work to be done for each item in the stream. + */ + void process(Closure<T> work); + + /** + * Returns a {@code Streamer} that will process the same stream as this streamer, but will stop + * processing when encountering the first item for which {@code cond} is true. + * + * @param cond a predicate that returns {@code false} as long as the stream should keep being + * processed. + * @return a streamer that will process items until the condition triggers. + */ + Streamer<T> endOn(Predicate<T> cond); + + /** + * Returns a {@code Streamer} that will process the same stream as this streamer, but with any + * items failing the filter to be omitted from processing. + * @param filter a predicate that returns {@code true} if an item in the stream should be + * processed + * @return a filtered streamer + */ + Streamer<T> filter(Predicate<T> filter); +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86a547b9/commons/src/main/java/com/twitter/common/io/ThriftCodec.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/com/twitter/common/io/ThriftCodec.java b/commons/src/main/java/com/twitter/common/io/ThriftCodec.java new file mode 100644 index 0000000..cf95bf6 --- /dev/null +++ b/commons/src/main/java/com/twitter/common/io/ThriftCodec.java @@ -0,0 +1,107 @@ +// ================================================================================================= +// Copyright 2011 Twitter, Inc. +// ------------------------------------------------------------------------------------------------- +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this work except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file, or at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ================================================================================================= + +package com.twitter.common.io; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Supplier; +import com.twitter.common.base.MoreSuppliers; +import org.apache.thrift.TBase; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TCompactProtocol; +import org.apache.thrift.protocol.TJSONProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.TIOStreamTransport; +import org.apache.thrift.transport.TTransport; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * A {@code Codec} that can encode and decode thrift structs. + */ +public class ThriftCodec<T extends TBase> implements Codec<T> { + + public static final Function<TTransport, TProtocol> JSON_PROTOCOL = + new Function<TTransport, TProtocol>() { + @Override public TProtocol apply(TTransport transport) { + return new TJSONProtocol(transport); + } + }; + + public static final Function<TTransport, TProtocol> BINARY_PROTOCOL = + new Function<TTransport, TProtocol>() { + @Override public TProtocol apply(TTransport transport) { + return new TBinaryProtocol(transport); + } + }; + + public static final Function<TTransport, TProtocol> COMPACT_PROTOCOL = + new Function<TTransport, TProtocol>() { + @Override public TProtocol apply(TTransport transport) { + return new TCompactProtocol(transport); + } + }; + + private final Supplier<T> templateSupplier; + private final Function<TTransport, TProtocol> protocolFactory; + + public static <T extends TBase> ThriftCodec<T> create(final Class<T> thriftStructType, + Function<TTransport, TProtocol> protocolFactory) { + return new ThriftCodec<T>(MoreSuppliers.of(thriftStructType), protocolFactory); + } + + /** + * @deprecated use {@link ThriftCodec#create(Class, Function)} instead. + */ + @Deprecated + public ThriftCodec(final Class<T> thriftStructType, + Function<TTransport, TProtocol> protocolFactory) { + this(MoreSuppliers.of(thriftStructType), protocolFactory); + } + + public ThriftCodec(Supplier<T> templateSupplier, + Function<TTransport, TProtocol> protocolFactory) { + this.templateSupplier = Preconditions.checkNotNull(templateSupplier); + this.protocolFactory = Preconditions.checkNotNull(protocolFactory); + } + + @Override + public void serialize(T item, OutputStream sink) throws IOException { + Preconditions.checkNotNull(item); + Preconditions.checkNotNull(sink); + try { + item.write(protocolFactory.apply(new TIOStreamTransport(null, sink))); + } catch (TException e) { + throw new IOException("Problem serializing thrift struct: " + item, e); + } + } + + @Override + public T deserialize(InputStream source) throws IOException { + Preconditions.checkNotNull(source); + T template = templateSupplier.get(); + try { + template.read(protocolFactory.apply(new TIOStreamTransport(source, null))); + } catch (TException e) { + throw new IOException("Problem de-serializing thrift struct from stream", e); + } + return template; + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86a547b9/commons/src/main/java/com/twitter/common/logging/BufferedLog.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/com/twitter/common/logging/BufferedLog.java b/commons/src/main/java/com/twitter/common/logging/BufferedLog.java new file mode 100644 index 0000000..54be868 --- /dev/null +++ b/commons/src/main/java/com/twitter/common/logging/BufferedLog.java @@ -0,0 +1,281 @@ +// ================================================================================================= +// Copyright 2011 Twitter, Inc. +// ------------------------------------------------------------------------------------------------- +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this work except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file, or at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ================================================================================================= + +package com.twitter.common.logging; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.twitter.common.quantity.Amount; +import com.twitter.common.quantity.Time; +import com.twitter.common.stats.StatImpl; +import com.twitter.common.stats.Stats; + +import java.util.List; +import java.util.Timer; +import java.util.TimerTask; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.logging.Logger; + +/** + * Log that buffers requests before sending them to a wrapped log. + * + * @author William Farner + */ +public class BufferedLog<T, R> implements Log<T, Void> { + private static final Logger LOG = Logger.getLogger(BufferedLog.class.getName()); + + private static final ExecutorService DEFAULT_EXECUTOR_SERVICE = + Executors.newSingleThreadExecutor( + new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Log Pusher-%d").build()); + private static final int DEFAULT_MAX_BUFFER_SIZE = 100000; + + // TODO(William Farner): Change to use a ScheduledExecutorService instead of a timer. + private final TimerTask logPusher = new TimerTask() { + @Override public void run() { + flush(); + } + }; + + // Local buffer of log messages. + private final List<T> localBuffer = Lists.newLinkedList(); + + // The log that is being buffered. + private Log<T, R> bufferedLog; + + // Filter to determine when a log request should be retried. + private Predicate<R> retryFilter = null; + + // Maximum number of log entries that can be buffered before truncation (lost messages). + private int maxBufferSize = DEFAULT_MAX_BUFFER_SIZE; + + // Maximum buffer length before attempting to submit. + private int chunkLength; + + // Maximum time for a message to sit in the buffer before attempting to flush. + private Amount<Integer, Time> flushInterval; + + // Service to handle flushing the log. + private ExecutorService logSubmitService = DEFAULT_EXECUTOR_SERVICE; + + private BufferedLog() { + // Created through builder. + + Stats.export(new StatImpl<Integer>("scribe_buffer_size") { + public Integer read() { return getBacklog(); } + }); + } + + public static <T, R> Builder<T, R> builder() { + return new Builder<T, R>(); + } + + /** + * Starts the log submission service by scheduling a timer to periodically submit messages. + */ + private void start() { + long flushIntervalMillis = flushInterval.as(Time.MILLISECONDS); + + new Timer(true).scheduleAtFixedRate(logPusher, flushIntervalMillis, flushIntervalMillis); + } + + /** + * Gets the current number of messages in the local buffer. + * + * @return The number of backlogged messages. + */ + protected int getBacklog() { + synchronized (localBuffer) { + return localBuffer.size(); + } + } + + /** + * Stores a log entry, flushing immediately if the buffer length limit is exceeded. + * + * @param entry Entry to log. + */ + @Override + public Void log(T entry) { + synchronized (localBuffer) { + localBuffer.add(entry); + + if (localBuffer.size() >= chunkLength) { + logSubmitService.submit(logPusher); + } + } + + return null; + } + + @Override + public Void log(List<T> entries) { + for (T entry : entries) log(entry); + + return null; + } + + @Override + public void flush() { + List<T> buffer = copyBuffer(); + if (buffer.isEmpty()) return; + + R result = bufferedLog.log(buffer); + + // Restore the buffer if the write was not successful. + if (retryFilter != null && retryFilter.apply(result)) { + LOG.warning("Log request failed, restoring spooled messages."); + restoreToLocalBuffer(buffer); + } + } + + /** + * Creats a snapshot of the local buffer and clears the local buffer. + * + * @return A snapshot of the local buffer. + */ + private List<T> copyBuffer() { + synchronized (localBuffer) { + List<T> bufferCopy = ImmutableList.copyOf(localBuffer); + localBuffer.clear(); + return bufferCopy; + } + } + + /** + * Restores log entries back to the local buffer. This can be used to commit entries back to the + * buffer after a flush operation failed. + * + * @param buffer The log entries to restore. + */ + private void restoreToLocalBuffer(List<T> buffer) { + synchronized (localBuffer) { + int restoreRecords = Math.min(buffer.size(), maxBufferSize - localBuffer.size()); + + if (restoreRecords != buffer.size()) { + LOG.severe((buffer.size() - restoreRecords) + " log records truncated!"); + + if (restoreRecords == 0) return; + } + + localBuffer.addAll(0, buffer.subList(buffer.size() - restoreRecords, buffer.size())); + } + } + + /** + * Configures a BufferedLog object. + * + * @param <T> Log message type. + * @param <R> Log result type. + */ + public static class Builder<T, R> { + private final BufferedLog<T, R> instance; + + public Builder() { + instance = new BufferedLog<T, R>(); + } + + /** + * Specifies the log that should be buffered. + * + * @param bufferedLog Log to buffer requests to. + * @return A reference to the builder. + */ + public Builder<T, R> buffer(Log<T, R> bufferedLog) { + instance.bufferedLog = bufferedLog; + return this; + } + + /** + * Adds a custom retry filter that will be used to determine whether a log result {@code R} + * should be used to indicate that a log request should be retried. Log submit retry behavior + * is not defined when the filter throws uncaught exceptions. + * + * @param retryFilter Filter to determine whether to retry. + * @return A reference to the builder. + */ + public Builder<T, R> withRetryFilter(Predicate<R> retryFilter) { + instance.retryFilter = retryFilter; + return this; + } + + /** + * Specifies the maximum allowable buffer size, after which log records will be dropped to + * conserve memory. + * + * @param maxBufferSize Maximum buffer size. + * @return A reference to the builder. + */ + public Builder<T, R> withMaxBuffer(int maxBufferSize) { + instance.maxBufferSize = maxBufferSize; + return this; + } + + /** + * Specifies the desired number of log records to submit in each request. + * + * @param chunkLength Maximum number of records to accumulate before trying to submit. + * @return A reference to the builder. + */ + public Builder<T, R> withChunkLength(int chunkLength) { + instance.chunkLength = chunkLength; + return this; + } + + /** + * Specifies the maximum amount of time that a log entry may wait in the buffer before an + * attempt is made to flush the buffer. + * + * @param flushInterval Log flush interval. + * @return A reference to the builder. + */ + public Builder<T, R> withFlushInterval(Amount<Integer, Time> flushInterval) { + instance.flushInterval = flushInterval; + return this; + } + + /** + * Specifies the executor service to use for (synchronously or asynchronously) sending + * log entries. + * + * @param logSubmitService Log submit executor service. + * @return A reference to the builder. + */ + public Builder<T, R> withExecutorService(ExecutorService logSubmitService) { + instance.logSubmitService = logSubmitService; + return this; + } + + /** + * Creates the buffered log. + * + * @return The prepared buffered log. + */ + public BufferedLog<T, R> build() { + Preconditions.checkArgument(instance.chunkLength > 0); + Preconditions.checkArgument(instance.flushInterval.as(Time.MILLISECONDS) > 0); + Preconditions.checkNotNull(instance.logSubmitService); + Preconditions.checkArgument(instance.chunkLength <= instance.maxBufferSize); + + instance.start(); + + return instance; + } + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86a547b9/commons/src/main/java/com/twitter/common/logging/Glog.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/com/twitter/common/logging/Glog.java b/commons/src/main/java/com/twitter/common/logging/Glog.java new file mode 100644 index 0000000..64d9334 --- /dev/null +++ b/commons/src/main/java/com/twitter/common/logging/Glog.java @@ -0,0 +1,211 @@ +// ================================================================================================= +// Copyright 2012 Twitter, Inc. +// ------------------------------------------------------------------------------------------------- +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this work except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file, or at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ================================================================================================= + +package com.twitter.common.logging; + +import javax.annotation.Nullable; + +import com.google.common.base.Objects; +import com.google.common.base.Throwables; + +import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +/** + * A utility that can format log records to match the format generated by glog: + * <pre> + * I0218 17:36:47.461 (source) (message) + * </pre> + */ +public final class Glog { + + /** + * Classifies the importance of a log message. + */ + public enum Level { + + /** + * Indicates the message's classification is unknown. This most likely indicates a + * configuration or programming error that can be corrected by mapping the underlying log + * system's level appropriately. + */ + UNKNOWN('U'), + + /** + * Indicates the message is for debugging purposes only. + */ + DEBUG('D'), + + /** + * Indicates a message of general interest. + */ + INFO('I'), + + /** + * Indicates a warning message likely worth of attention. + */ + WARNING('W'), + + /** + * Indicates an unexpected error. + */ + ERROR('E'), + + /** + * Indicates a fatal exception generally paired with actions to shut down the errored process. + */ + FATAL('F'); + + final char label; + + private Level(char label) { + this.label = label; + } + } + + /** + * An object that can provide details of a log record. + * + * @param <T> The type of log record the formatter handles. + */ + public interface Formatter<T> { + + /** + * Gets the message contained in the log record. + * + * @param record The record to extract a message from. + * @return The formatted message. + */ + String getMessage(T record); + + /** + * Gets the class name of the class that sent the log record for logging. + * + * @param record The record to extract a producing class name from. + * @return The producing class if known; otherwise {@code null}. + */ + @Nullable + String getClassName(T record); + + /** + * Gets the name of the method of within the class that sent the log record for logging. + * + * @param record The record to extract a producing method name from. + * @return The producing method name if known; otherwise {@code null}. + */ + @Nullable + String getMethodName(T record); + + /** + * Gets the level of the log record. + * + * @param record The record to extract a log level from. + * @return The record's log level. Can be {@code null} or {@link Level#UNKNOWN} if unknown. + */ + @Nullable + Level getLevel(T record); + + /** + * Gets the timestamp in milliseconds since the epoch when the log record was generated. + * + * @param record The record to extract a time stamp from. + * @return The log record's birth date. + */ + long getTimeStamp(T record); + + /** + * Gets the id of the thread that generated the log record. + * + * @param record The record to extract a thread id from. + * @return The id of the thread that generated the log record. + */ + long getThreadId(T record); + + /** + * Gets the exception associated with the log record if any. + * + * @param record The record to extract an exception from. + * @return The exception associated with the log record; may be {@code null}. + */ + @Nullable + Throwable getThrowable(T record); + } + + private static final DateTimeFormatter DATE_TIME_FORMATTER = + DateTimeFormat.forPattern("MMdd HH:mm:ss.SSS").withZone(DateTimeZone.UTC); + + private static final int BASE_MESSAGE_LENGTH = + 1 // Level char. + + 4 // Month + day + + 1 // space + + 12 // Timestamp + + 1 // space + + 6 // THREAD + + 4 // Room for thread ID. + + 1; // space + + /** + * Converts the given log record into a glog format log line using the given formatter. + * + * @param formatter A formatter that understands how to unpack the given log record. + * @param record A structure containing log data. + * @param <T> The type of log record. + * @return A glog formatted log line. + */ + public static <T> String formatRecord(Formatter<T> formatter, T record) { + String message = formatter.getMessage(record); + int messageLength = BASE_MESSAGE_LENGTH + + 2 // Colon and space + + message.length(); + + String className = formatter.getClassName(record); + String methodName = null; + if (className != null) { + messageLength += className.length(); + methodName = formatter.getMethodName(record); + if (methodName != null) { + messageLength += 1; // Period between class and method. + messageLength += methodName.length(); + } + } + + StringBuilder sb = new StringBuilder(messageLength) + .append(Objects.firstNonNull(formatter.getLevel(record), Level.UNKNOWN).label) + .append(DATE_TIME_FORMATTER.print(formatter.getTimeStamp(record))) + .append(" THREAD") + .append(formatter.getThreadId(record)); + + if (className != null) { + sb.append(' ').append(className); + if (methodName != null) { + sb.append('.').append(methodName); + } + } + + sb.append(": ").append(message); + Throwable throwable = formatter.getThrowable(record); + if (throwable != null) { + sb.append('\n').append(Throwables.getStackTraceAsString(throwable)); + } + + return sb.append('\n').toString(); + } + + private Glog() { + // utility + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86a547b9/commons/src/main/java/com/twitter/common/logging/Log.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/com/twitter/common/logging/Log.java b/commons/src/main/java/com/twitter/common/logging/Log.java new file mode 100644 index 0000000..3ba4150 --- /dev/null +++ b/commons/src/main/java/com/twitter/common/logging/Log.java @@ -0,0 +1,48 @@ +// ================================================================================================= +// Copyright 2011 Twitter, Inc. +// ------------------------------------------------------------------------------------------------- +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this work except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file, or at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ================================================================================================= + +package com.twitter.common.logging; + +import java.util.List; + +/** + * Logs messages to scribe. + * + * @author William Farner + */ +public interface Log<T, R> { + + /** + * Submits a log message. + * + * @param entry Entry to log. + * @return The result of the log request. + */ + public R log(T entry); + + /** + * Batch version of log. + * + * @param entries Entries to log. + * @return The result of the log request. + */ + public R log(List<T> entries); + + /** + * Flushes the log, attempting to purge any state that is only stored locally. + */ + public void flush(); +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86a547b9/commons/src/main/java/com/twitter/common/logging/LogFormatter.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/com/twitter/common/logging/LogFormatter.java b/commons/src/main/java/com/twitter/common/logging/LogFormatter.java new file mode 100644 index 0000000..6c27188 --- /dev/null +++ b/commons/src/main/java/com/twitter/common/logging/LogFormatter.java @@ -0,0 +1,67 @@ +package com.twitter.common.logging; + +import java.util.logging.Formatter; +import java.util.logging.Level; +import java.util.logging.LogRecord; + +import com.google.common.collect.ImmutableMap; + +/** + * Log formatter to match the format generated by glog. + * + * @see Glog + */ +public class LogFormatter extends Formatter implements Glog.Formatter<LogRecord> { + + private static final ImmutableMap<Level, Glog.Level> LEVEL_LABELS = + ImmutableMap.<Level, Glog.Level>builder() + .put(Level.FINEST, Glog.Level.DEBUG) + .put(Level.FINER, Glog.Level.DEBUG) + .put(Level.FINE, Glog.Level.DEBUG) + .put(Level.CONFIG, Glog.Level.INFO) + .put(Level.INFO, Glog.Level.INFO) + .put(Level.WARNING, Glog.Level.WARNING) + .put(Level.SEVERE, Glog.Level.ERROR) + .build(); + + + @Override + public String format(final LogRecord record) { + return Glog.formatRecord(this, record); + } + + @Override + public String getMessage(LogRecord record) { + return formatMessage(record); + } + + @Override + public String getClassName(LogRecord record) { + return record.getSourceClassName(); + } + + @Override + public String getMethodName(LogRecord record) { + return record.getSourceMethodName(); + } + + @Override + public Glog.Level getLevel(LogRecord record) { + return LEVEL_LABELS.get(record.getLevel()); + } + + @Override + public long getTimeStamp(LogRecord record) { + return record.getMillis(); + } + + @Override + public long getThreadId(LogRecord record) { + return record.getThreadID(); + } + + @Override + public Throwable getThrowable(LogRecord record) { + return record.getThrown(); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86a547b9/commons/src/main/java/com/twitter/common/logging/LogUtil.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/com/twitter/common/logging/LogUtil.java b/commons/src/main/java/com/twitter/common/logging/LogUtil.java new file mode 100644 index 0000000..f18b705 --- /dev/null +++ b/commons/src/main/java/com/twitter/common/logging/LogUtil.java @@ -0,0 +1,93 @@ +// ================================================================================================= +// Copyright 2011 Twitter, Inc. +// ------------------------------------------------------------------------------------------------- +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this work except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file, or at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ================================================================================================= + +package com.twitter.common.logging; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.SystemUtils; + +import java.io.File; +import java.util.logging.LogManager; +import java.util.logging.Logger; + +/** + * Logging utility functions. + * + * @author William Farner + */ +public class LogUtil { + + private static final Logger LOG = Logger.getLogger(LogUtil.class.getName()); + + private static final String LOG_MANAGER_FILE_PROP = "java.util.logging.FileHandler.pattern"; + + @VisibleForTesting + static final File DEFAULT_LOG_DIR = new File("/var/log"); + + /** + * Gets the log directory as configured with the log manager. This will attempt to expand any + * directory wildcards that are included in log file property. + * + * @return The configured log directory. + */ + public static File getLogManagerLogDir() { + return getLogManagerLogDir(LogManager.getLogManager().getProperty(LOG_MANAGER_FILE_PROP)); + } + + /** + * Gets the log directory as specified in a log file pattern. This will attempt to expand any + * directory wildcards that are included in log file property. + * + * @param logFilePattern The pattern to extract the log directory from. + * @return The configured log directory. + */ + public static File getLogManagerLogDir(String logFilePattern) { + if (StringUtils.isEmpty(logFilePattern)) { + LOG.warning("Could not find log dir in logging property " + LOG_MANAGER_FILE_PROP + + ", reading from " + DEFAULT_LOG_DIR); + return DEFAULT_LOG_DIR; + } + + String logDir = expandWildcard(logFilePattern, "%t", SystemUtils.JAVA_IO_TMPDIR); + logDir = expandWildcard(logDir, "%h", SystemUtils.USER_HOME); + File parent = new File(logDir).getParentFile(); + return parent == null ? new File(".") : parent; + } + + /** + * Expands a directory path wildcard within a file pattern string. + * Correctly handles cases where the replacement string does and does not contain a trailing + * slash. + * + * @param pattern File pattern string, which may or may not contain a wildcard. + * @param dirWildcard Wildcard string to expand. + * @param replacement Path component to expand wildcard to. + * @return {@code pattern} with all instances of {@code dirWildcard} replaced with + * {@code replacement}. + */ + private static String expandWildcard(String pattern, String dirWildcard, String replacement) { + String replace = dirWildcard; + if (replacement.charAt(replacement.length() - 1) == '/') { + replace += '/'; + } + return pattern.replaceAll(replace, replacement); + } + + private LogUtil() { + // Utility class. + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86a547b9/commons/src/main/java/com/twitter/common/logging/RootLogConfig.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/com/twitter/common/logging/RootLogConfig.java b/commons/src/main/java/com/twitter/common/logging/RootLogConfig.java new file mode 100644 index 0000000..098ee44 --- /dev/null +++ b/commons/src/main/java/com/twitter/common/logging/RootLogConfig.java @@ -0,0 +1,326 @@ +package com.twitter.common.logging; + +import java.util.HashMap; +import java.util.Map; +import java.util.logging.ConsoleHandler; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogManager; +import java.util.logging.Logger; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; + +import com.twitter.common.args.Arg; +import com.twitter.common.args.CmdLine; + +/** + * A configuration class for the root java.util.logging Logger. + * + * Defines flags to control the behavior behavior of the root logger similarly to Google's glog + * library (see http://code.google.com/p/google-glog ). + */ +public class RootLogConfig { + /** + * An enum reflecting log {@link Level} constants. + */ + public enum LogLevel { + FINEST(Level.FINEST), + FINER(Level.FINER), + FINE(Level.FINE), + CONFIG(Level.CONFIG), + INFO(Level.INFO), + WARNING(Level.WARNING), + SEVERE(Level.SEVERE); + + private final Level level; + + private LogLevel(Level level) { + this.level = level; + } + + private Level getLevel() { + return level; + } + } + + @CmdLine(name = "logtostderr", help = "Log messages to stderr instead of logfiles.") + private static Arg<Boolean> LOGTOSTDERR = Arg.create(false); + + @CmdLine(name = "alsologtostderr", + help = "Log messages to stderr, in addition to log files. Ignored when --logtostderr") + private static Arg<Boolean> ALSOLOGTOSTDERR = Arg.create(false); + + @CmdLine(name = "vlog", + help = "The value is one of the constants in java.util.logging.Level. " + + "Shows all messages with level equal or higher " + + "than the value of this flag.") + private static Arg<LogLevel> VLOG = Arg.create(LogLevel.INFO); + + @CmdLine(name = "vmodule", + help = "Per-class verbose level. The argument has to contain a comma-separated list " + + "of <class_name>=<log_level>. <class_name> is the full-qualified name of a " + + "class, <log_level> is one of the constants in java.util.logging.Level. " + + "<log_level> overrides any value given by --vlog.") + private static Arg<Map<Class<?>, LogLevel>> VMODULE = + Arg.<Map<Class<?>, LogLevel>>create(new HashMap<Class<?>, LogLevel>()); + + @CmdLine(name = "use_glog_formatter", help = "True to use the glog formatter exclusively.") + private static Arg<Boolean> USE_GLOG_FORMATTER = Arg.create(true); + + /** + * Represents a logging configuration for java.util.logging. + */ + public static final class Configuration { + boolean logToStderr = false; + boolean alsoLogToStderr = false; + boolean useGLogFormatter = true; + LogLevel vlog = LogLevel.INFO; + ImmutableMap<Class<?>, LogLevel> vmodule = ImmutableMap.of(); + String rootLoggerName = ""; + + Configuration() { + // Guard for internal use only. + } + + /** + * Returns {@code true} if configured to log just to stderr. + */ + public boolean isLogToStderr() { + return logToStderr; + } + + /** + * Returns {@code true} if configured to log to stderr in addition to log files.. + */ + public boolean isAlsoLogToStderr() { + return alsoLogToStderr; + } + + /** + * Returns {@code true} if configured to log in google-glog format. + */ + public boolean isUseGLogFormatter() { + return useGLogFormatter; + } + + /** + * Returns the default global log level. + */ + public LogLevel getVlog() { + return vlog; + } + + /** + * Returns the custom log levels configured for individual classes. + */ + public ImmutableMap<Class<?>, LogLevel> getVmodule() { + return vmodule; + } + + /** + * Applies this configuration to the root log. + */ + public void apply() { + RootLogConfig.configure(this); + } + } + + /** + * A builder-pattern class used to perform the configuration programmatically + * (i.e. not through flags). + * Example: + * <code> + * RootLogConfig.builder().logToStderr(true).build().apply(); + * </code> + */ + public static final class Builder { + private final Configuration configuration; + + private Builder() { + this.configuration = new Configuration(); + } + + /** + * Only log messages to stderr, instead of log files. Overrides alsologtostderr. + * Default: false. + * + * @param flag True to enable, false to disable. + * @return this Configuration object. + */ + public Builder logToStderr(boolean flag) { + configuration.logToStderr = flag; + return this; + } + + /** + * Also log messages to stderr, in addition to log files. + * Overridden by logtostderr. + * Default: false. + * + * @param flag True to enable, false to disable. + * @return this Configuration object. + */ + public Builder alsoLogToStderr(boolean flag) { + configuration.alsoLogToStderr = flag; + return this; + } + + /** + * Format log messages in one-line with a header, similar to google-glog. + * Default: false. + * + * @param flag True to enable, false to disable. + * @return this Configuration object. + */ + public Builder useGLogFormatter(boolean flag) { + configuration.useGLogFormatter = flag; + return this; + } + + /** + * Output log messages at least at the given verbosity level. + * Overridden by vmodule. + * Default: INFO + * + * @param level LogLevel enumerator for the minimum log message verbosity level that is output. + * @return this Configuration object. + */ + public Builder vlog(LogLevel level) { + Preconditions.checkNotNull(level); + configuration.vlog = level; + return this; + } + + /** + * Output log messages for a given set of classes at the associated verbosity levels. + * Overrides vlog. + * Default: no classes are treated specially. + * + * @param pairs Map of classes and correspoding log levels. + * @return this Configuration object. + */ + public Builder vmodule(Map<Class<?>, LogLevel> pairs) { + Preconditions.checkNotNull(pairs); + configuration.vmodule = ImmutableMap.copyOf(pairs); + return this; + } + + /** + * Returns the built up configuration. + */ + public Configuration build() { + return configuration; + } + + // Intercepts the root logger, for testing purposes only. + @VisibleForTesting + Builder rootLoggerName(String name) { + Preconditions.checkNotNull(name); + Preconditions.checkArgument(!name.isEmpty()); + configuration.rootLoggerName = name; + return this; + } + } + + /** + * Creates a new Configuration builder object. + * + * @return The builder. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * Creates a logging configuration using flags + * + * @return The logging configuration specified via command line flags. + */ + public static Configuration configurationFromFlags() { + return builder() + .logToStderr(LOGTOSTDERR.get()) + .alsoLogToStderr(ALSOLOGTOSTDERR.get()) + .useGLogFormatter(USE_GLOG_FORMATTER.get()) + .vlog(VLOG.get()) + .vmodule(VMODULE.get()) + .build(); + } + + private static void configure(Configuration configuration) { + // Edit the properties of the root logger. + Logger rootLogger = Logger.getLogger(configuration.rootLoggerName); + if (configuration.logToStderr) { + setLoggerToStderr(rootLogger); + } else if (configuration.alsoLogToStderr) { + setLoggerToAlsoStderr(rootLogger); + } + if (configuration.useGLogFormatter) { + setGLogFormatter(rootLogger); + } + if (configuration.vlog != null) { + setVlog(rootLogger, configuration.vlog); + } + if (configuration.vmodule != null) { + setVmodules(configuration.vmodule); + } + } + + private static void setLoggerToStderr(Logger logger) { + LogManager.getLogManager().reset(); + setConsoleHandler(logger, true); + } + + private static void setLoggerToAlsoStderr(Logger logger) { + setConsoleHandler(logger, false); + } + + private static void setConsoleHandler(Logger logger, boolean removeOtherHandlers) { + Handler consoleHandler = null; + for (Handler h : logger.getHandlers()) { + if (h instanceof ConsoleHandler) { + consoleHandler = h; + } else if (removeOtherHandlers) { + logger.removeHandler(h); + } + } + if (consoleHandler == null) { + consoleHandler = new ConsoleHandler(); + logger.addHandler(new ConsoleHandler()); + } + consoleHandler.setLevel(Level.ALL); + consoleHandler.setFilter(null); + } + + private static void setGLogFormatter(Logger logger) { + for (Handler h : logger.getHandlers()) { + h.setFormatter(new LogFormatter()); + } + } + + private static void setVmodules(Map<Class<?>, LogLevel> vmodules) { + for (Map.Entry<Class<?>, LogLevel> entry : vmodules.entrySet()) { + String className = entry.getKey().getName(); + Logger logger = Logger.getLogger(className); + setVlog(logger, entry.getValue()); + } + } + + private static void setVlog(Logger logger, LogLevel logLevel) { + final Level newLevel = logLevel.getLevel(); + logger.setLevel(newLevel); + do { + for (Handler handler : logger.getHandlers()) { + Level handlerLevel = handler.getLevel(); + if (newLevel.intValue() < handlerLevel.intValue()) { + handler.setLevel(newLevel); + } + } + } while (logger.getUseParentHandlers() && (logger = logger.getParent()) != null); + } + + // Utility class. + private RootLogConfig() { + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86a547b9/commons/src/main/java/com/twitter/common/logging/julbridge/JULBridgeHandler.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/com/twitter/common/logging/julbridge/JULBridgeHandler.java b/commons/src/main/java/com/twitter/common/logging/julbridge/JULBridgeHandler.java new file mode 100644 index 0000000..7340312 --- /dev/null +++ b/commons/src/main/java/com/twitter/common/logging/julbridge/JULBridgeHandler.java @@ -0,0 +1,199 @@ +// ================================================================================================= +// Copyright 2013 Twitter, Inc. +// ------------------------------------------------------------------------------------------------- +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this work except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file, or at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ================================================================================================= + +package com.twitter.common.logging.julbridge; + +import java.text.MessageFormat; +import java.util.MissingResourceException; +import java.util.logging.Formatter; +import java.util.logging.Handler; +import java.util.logging.LogRecord; + +import javax.annotation.Nullable; + +import org.apache.log4j.Level; +import org.apache.log4j.LogManager; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.LocationInfo; +import org.apache.log4j.spi.LoggerRepository; +import org.apache.log4j.spi.LoggingEvent; +import org.apache.log4j.spi.ThrowableInformation; + +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * JUL Handler to convert JUL {@link LogRecord} messages into Log4j's {@link LoggingEvent} messages, + * and route them to a Log4J logger with the same name as the JUL logger. + */ +public class JULBridgeHandler extends Handler { + private static final String UNKNOWN_LOGGERNAME = "unknown"; + + /** + * Converts a JUL log record into a Log4J logging event. + * + * @param record the JUL log record to convert + * @param logger the Log4J logger to use for the logging event + * @param level the Log4J level to use for the logging event + * @param useExtendedLocationInfo if false, do no try to get source file and line informations + * @return a Log4J logging event + */ + static LoggingEvent toLoggingEvent(LogRecord record, Logger logger, Level level, + boolean useExtendedLocationInfo) { + + LocationInfo locationInfo = useExtendedLocationInfo + ? new LocationInfo(new Throwable(), record.getSourceClassName()) + : new LocationInfo("?", record.getSourceClassName(), record.getSourceMethodName(), "?"); + + // Getting thread name from thread id? complicated... + String threadName = String.valueOf(record.getThreadID()); + ThrowableInformation throwableInformation = record.getThrown() == null + ? null + : new ThrowableInformation(record.getThrown()); + + return new LoggingEvent( + record.getSourceClassName(), + logger, + record.getMillis(), + level, + formatMessage(record), + threadName, + throwableInformation, + null /* ndc */, + locationInfo, + null /* properties */); + } + + /** + * Formats a log record message in a way similar to {@link Formatter#formatMessage(LogRecord)}. + * + * If the record contains a resource bundle, a lookup is done to find a localized version. + * + * If the record contains parameters, the message is formatted using + * {@link MessageFormat#format(String, Object...)} + * + * @param record the log record used to format the message + * @return a formatted string + */ + static String formatMessage(LogRecord record) { + String message = record.getMessage(); + + // Look for a resource bundle + java.util.ResourceBundle catalog = record.getResourceBundle(); + if (catalog != null) { + try { + message = catalog.getString(record.getMessage()); + } catch (MissingResourceException e) { + // Not found? Fallback to original message string + message = record.getMessage(); + } + } + + Object parameters[] = record.getParameters(); + if (parameters == null || parameters.length == 0) { + // No parameters? just return the message string + return message; + } + + // Try formatting + try { + return MessageFormat.format(message, parameters); + } catch (IllegalArgumentException e) { + return message; + } + } + + private final LoggerRepository loggerRepository; + private final boolean useExtendedLocationInfo; + + /** + * Creates a new JUL handler. Equivalent to calling {@link #JULBridgeHandler(boolean)} passing + * <code>false</code> as argument. + */ + public JULBridgeHandler() { + this(LogManager.getLoggerRepository(), false); + } + + /** + * Creates a new JUL handler. + * Equivalent to calling {@link #JULBridgeHandler(LoggerRepository, boolean)} passing + * <code>LogManager.getLoggerRepository()</code> and <code>useExtendedLocationInfo</code> as + * arguments. + * + * @param useExtendedLocationInfo if true, try to add source filename and line info to log message + */ + public JULBridgeHandler(boolean useExtendedLocationInfo) { + this(LogManager.getLoggerRepository(), useExtendedLocationInfo); + } + + /** + * Creates a new JUL handler. + * + * @param loggerRepository Log4j logger repository where to get loggers from + * @param useExtendedLocationInfo if true, try to add source filename and line info to log message + * @throws NullPointerException if loggerRepository is null + */ + public JULBridgeHandler(LoggerRepository loggerRepository, boolean useExtendedLocationInfo) { + this.loggerRepository = checkNotNull(loggerRepository); + this.useExtendedLocationInfo = useExtendedLocationInfo; + } + + /** + * Gets a Log4J Logger with the same name as the logger name stored in the log record. + * + * @param record a JUL log record + * @return a Log4J logger with the same name, or name {@value #UNKNOWN_LOGGERNAME} if no name is + * present in the record. + */ + Logger getLogger(LogRecord record) { + String loggerName = record.getLoggerName(); + if (loggerName == null) { + loggerName = UNKNOWN_LOGGERNAME; + } + + return loggerRepository.getLogger(loggerName); + } + + /** + * Publishes the log record to a Log4J logger of the same name. + * + * Before formatting the message, level is converted and message is discarded if Log4j logger is + * not enabled for that level. + * + * @param record the record to publish + */ + @Override + public void publish(@Nullable LogRecord record) { + // Ignore silently null records + if (record == null) { + return; + } + + Logger log4jLogger = getLogger(record); + Level log4jLevel = JULBridgeLevelConverter.toLog4jLevel(record.getLevel()); + + if (log4jLogger.isEnabledFor(log4jLevel)) { + LoggingEvent event = toLoggingEvent(record, log4jLogger, log4jLevel, useExtendedLocationInfo); + + log4jLogger.callAppenders(event); + } + } + + @Override + public void flush() {} + + @Override + public void close() {} +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86a547b9/commons/src/main/java/com/twitter/common/logging/julbridge/JULBridgeLevelConverter.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/com/twitter/common/logging/julbridge/JULBridgeLevelConverter.java b/commons/src/main/java/com/twitter/common/logging/julbridge/JULBridgeLevelConverter.java new file mode 100644 index 0000000..6a4393e --- /dev/null +++ b/commons/src/main/java/com/twitter/common/logging/julbridge/JULBridgeLevelConverter.java @@ -0,0 +1,102 @@ +// ================================================================================================= +// Copyright 2013 Twitter, Inc. +// ------------------------------------------------------------------------------------------------- +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this work except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file, or at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ================================================================================================= + +package com.twitter.common.logging.julbridge; + +import java.util.logging.Level; + +import static com.google.common.base.Preconditions.checkNotNull; + +/** + * An utility class to convert between JUL and Log4j Levels. Mapping is as follows: + * <ul> + * <li>FINEST <-> TRACE</li> + * <li>FINER -> DEBUG</li> + * <li>FINE <-> DEBUG</li> + * <li>INFO <-> INFO</li> + * <li>WARNING <-> WARN</li> + * <li>SEVERE <-> ERROR</li> + * <li>SEVERE <- FATAL</li> + * </ul> + * + * Unknowns levels are mapped to FINE/DEBUG + */ +public class JULBridgeLevelConverter { + + private JULBridgeLevelConverter() {} + + /** + * Converts a JUL level into a Log4j level. + * + * @param level the JUL level to convert + * @return a Log4j level + * @throws NullPointerException if level is null + */ + public static org.apache.log4j.Level toLog4jLevel(Level level) { + checkNotNull(level); + + if (level == Level.FINEST) { + return org.apache.log4j.Level.TRACE; + } else if (level == Level.FINER) { + return org.apache.log4j.Level.DEBUG; + } else if (level == Level.FINE) { + return org.apache.log4j.Level.DEBUG; + } else if (level == Level.INFO) { + return org.apache.log4j.Level.INFO; + } else if (level == Level.WARNING) { + return org.apache.log4j.Level.WARN; + } else if (level == Level.SEVERE) { + return org.apache.log4j.Level.ERROR; + } else if (level == Level.ALL) { + return org.apache.log4j.Level.ALL; + } else if (level == Level.OFF) { + return org.apache.log4j.Level.OFF; + } + + return org.apache.log4j.Level.DEBUG; + } + + /** + * Converts a Log4j level into a JUL level. + * + * @param level the Log4j level to convert + * @return a JUL level + * @throws NullPointerException if level is null + */ + public static Level fromLog4jLevel(org.apache.log4j.Level level) { + checkNotNull(level); + + if (level == org.apache.log4j.Level.TRACE) { + return Level.FINEST; + } else if (level == org.apache.log4j.Level.DEBUG) { + return Level.FINE; + } else if (level == org.apache.log4j.Level.INFO) { + return Level.INFO; + } else if (level == org.apache.log4j.Level.WARN) { + return Level.WARNING; + } else if (level == org.apache.log4j.Level.ERROR) { + return Level.SEVERE; + } else if (level == org.apache.log4j.Level.FATAL) { + return Level.SEVERE; + } else if (level == org.apache.log4j.Level.ALL) { + return Level.ALL; + } else if (level == org.apache.log4j.Level.OFF) { + return Level.OFF; + } + + return Level.FINE; + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86a547b9/commons/src/main/java/com/twitter/common/logging/julbridge/JULBridgeLogManager.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/com/twitter/common/logging/julbridge/JULBridgeLogManager.java b/commons/src/main/java/com/twitter/common/logging/julbridge/JULBridgeLogManager.java new file mode 100644 index 0000000..e41a166 --- /dev/null +++ b/commons/src/main/java/com/twitter/common/logging/julbridge/JULBridgeLogManager.java @@ -0,0 +1,89 @@ +// ================================================================================================= +// Copyright 2013 Twitter, Inc. +// ------------------------------------------------------------------------------------------------- +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this work except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file, or at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ================================================================================================= +package com.twitter.common.logging.julbridge; + +import java.util.logging.LogManager; +import java.util.logging.Logger; + +import org.apache.log4j.spi.LoggerRepository; + +/** + * A JUL LogManager which takes over the logging configuration and redirects all messages to Log4j. + * + * The approach is inspired by the apache-jul-log4j-bridge project from Paul Smith + * (<[email protected]>) and available at <a + * href="http://people.apache.org/~psmith/logging.apache.org/sandbox/jul-log4j-bridge/"; /> + * + * During initialization, it resets configuration and adds a default handler to the root logger to + * perform the redirection. It also sets the root logger level to the Log4j repository threshold. + * This implies that Log4j is properly configured before this manager is taking over. + * + * To install this log manager, simply add the following property to the java command line: + * <code>-Djava.util.logging.manager=com.twitter.common.logging.julbridge.JULBridgeLogManager</code> + * + * It is possible to configure using extended location information (source filename and line info) + * by adding the following property to the java command line: + * <code>-Dcom.twitter.common.logging.julbridge.use-extended-location-info=true</code> + * + */ +public final class JULBridgeLogManager extends LogManager { + /** + * System property name to control if log messages sent from JUL to log4j should contain + * extended location information. + * + * Set @value to true to add source filename and line number to each message. + */ + public static final String USE_EXTENDED_LOCATION_INFO_PROPERTYNAME = + "com.twitter.common.logging.julbridge.use-extended-location-info"; + + /* + * LogManager requires a public no-arg constructor to be present so a new instance can be created + * when configured using the system property. A private constructor will throw an exception. + */ + public JULBridgeLogManager() {} + + @Override + public void readConfiguration() { + assimilate(org.apache.log4j.LogManager.getLoggerRepository()); + } + + /** + * Assimilates an existing JUL log manager. Equivalent to calling + * {@link #assimilate(LoggerRepository)} with <code>LogManager.getLoggerRepository</code>. + */ + public static void assimilate() { + assimilate(org.apache.log4j.LogManager.getLoggerRepository()); + } + + /** + * Assimilates an existing JUL log manager. + * + * It resets the manager configuration, and adds a bridge handler to the root logger. Messages are + * redirected to the specified Log4j logger repository. + * + * @param loggerRepository the Log4j logger repository to use to redirect messages + */ + public static void assimilate(LoggerRepository loggerRepository) { + LogManager.getLogManager().reset(); + + boolean withExtendedLocationInfos = + Boolean.getBoolean(USE_EXTENDED_LOCATION_INFO_PROPERTYNAME); + + Logger rootLogger = Logger.getLogger(""); + rootLogger.setLevel(JULBridgeLevelConverter.fromLog4jLevel(loggerRepository.getThreshold())); + rootLogger.addHandler(new JULBridgeHandler(loggerRepository, withExtendedLocationInfos)); + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86a547b9/commons/src/main/java/com/twitter/common/logging/log4j/GlogLayout.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/com/twitter/common/logging/log4j/GlogLayout.java b/commons/src/main/java/com/twitter/common/logging/log4j/GlogLayout.java new file mode 100644 index 0000000..fb2d908 --- /dev/null +++ b/commons/src/main/java/com/twitter/common/logging/log4j/GlogLayout.java @@ -0,0 +1,103 @@ +// ================================================================================================= +// Copyright 2012 Twitter, Inc. +// ------------------------------------------------------------------------------------------------- +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this work except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file, or at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ================================================================================================= + +package com.twitter.common.logging.log4j; + +import com.google.common.collect.ImmutableMap; + +import org.apache.log4j.Layout; +import org.apache.log4j.Level; +import org.apache.log4j.spi.LocationInfo; +import org.apache.log4j.spi.LoggingEvent; +import org.apache.log4j.spi.ThrowableInformation; + +import com.twitter.common.logging.Glog; +import com.twitter.common.logging.Glog.Formatter; + +/** + * Log4j Layout to match the format generated by glog. + * + * @see Glog + */ +public class GlogLayout extends Layout implements Formatter<LoggingEvent> { + + private static final ImmutableMap<Level, Glog.Level> LEVEL_LABELS = + ImmutableMap.<Level, Glog.Level>builder() + .put(Level.TRACE, Glog.Level.DEBUG) + .put(Level.DEBUG, Glog.Level.DEBUG) + .put(Level.INFO, Glog.Level.INFO) + .put(Level.WARN, Glog.Level.WARNING) + .put(Level.ERROR, Glog.Level.ERROR) + .put(Level.FATAL, Glog.Level.FATAL) + .build(); + + @Override + public String format(LoggingEvent record) { + return Glog.formatRecord(this, record); + } + + @Override + public boolean ignoresThrowable() { + return false; // We handle stack trace formatting. + } + + @Override + public void activateOptions() { + // We use no options + } + + @Override + public String getMessage(LoggingEvent record) { + return record.getRenderedMessage(); + } + + @Override + public String getClassName(LoggingEvent record) { + LocationInfo locationInformation = record.getLocationInformation(); + return (locationInformation != null) + ? locationInformation.getClassName() + : null; + } + + @Override + public String getMethodName(LoggingEvent record) { + LocationInfo locationInformation = record.getLocationInformation(); + return (locationInformation != null) + ? record.getLocationInformation().getMethodName() + : null; + } + + @Override + public Glog.Level getLevel(LoggingEvent record) { + return LEVEL_LABELS.get(record.getLevel()); + } + + @Override + public long getTimeStamp(LoggingEvent record) { + return record.getTimeStamp(); + } + + @Override + public long getThreadId(LoggingEvent record) { + return Thread.currentThread().getId(); + } + + @Override + public Throwable getThrowable(LoggingEvent record) { + ThrowableInformation throwableInformation = record.getThrowableInformation(); + return throwableInformation != null ? throwableInformation.getThrowable() : null; + } +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86a547b9/commons/src/main/java/com/twitter/common/net/Environment.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/com/twitter/common/net/Environment.java b/commons/src/main/java/com/twitter/common/net/Environment.java new file mode 100644 index 0000000..68de913 --- /dev/null +++ b/commons/src/main/java/com/twitter/common/net/Environment.java @@ -0,0 +1,50 @@ +// ================================================================================================= +// Copyright 2011 Twitter, Inc. +// ------------------------------------------------------------------------------------------------- +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this work except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file, or at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ================================================================================================= + +package com.twitter.common.net; + +/** + * Represents a network environment at the granularity of a datacenter. + * + * @author John Sirois + */ +public interface Environment { + + /** + * Returns the name of this network environment's datacenter. + * + * @return the name of this environment's datacenter + */ + String dcName(); + + /** + * Creates a fully qualified hostname for a given unqualified hostname in the network + * environment's datacenter. Does not confirm that the host exists. + * + * @param hostname The simple hostname to qualify. + * @return The fully qualified hostname. + */ + String fullyQualify(String hostname); + + /** + * Checks if a given {@code hostname} is a valid hostname for a host in this network environment; + * does not guarantee that the host exists in this network environment. + * + * @param hostname The simple hostname to check for membership in this network environment. + * @return {@code true} if the hostname is a valid hostname for this network environment. + */ + boolean contains(String hostname); +} http://git-wip-us.apache.org/repos/asf/aurora/blob/86a547b9/commons/src/main/java/com/twitter/common/net/InetSocketAddressHelper.java ---------------------------------------------------------------------- diff --git a/commons/src/main/java/com/twitter/common/net/InetSocketAddressHelper.java b/commons/src/main/java/com/twitter/common/net/InetSocketAddressHelper.java new file mode 100644 index 0000000..5c970e0 --- /dev/null +++ b/commons/src/main/java/com/twitter/common/net/InetSocketAddressHelper.java @@ -0,0 +1,139 @@ +// ================================================================================================= +// Copyright 2011 Twitter, Inc. +// ------------------------------------------------------------------------------------------------- +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this work except in compliance with the License. +// You may obtain a copy of the License in the LICENSE file, or at: +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ================================================================================================= + +package com.twitter.common.net; + +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; +import org.apache.commons.lang.StringUtils; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.util.Set; + +/** + * A utility that can parse [host]:[port] pairs or :[port] designators into instances of + * {@link java.net.InetSocketAddress}. The literal '*' can be specified for port as an alternative + * to '0' to indicate any local port. + * + * @author John Sirois + */ +public final class InetSocketAddressHelper { + + /** + * A function that uses {@link #parse(String)} to map an endpoint spec to an + * {@link InetSocketAddress}. + */ + public static final Function<String, InetSocketAddress> STR_TO_INET = + new Function<String, InetSocketAddress>() { + @Override public InetSocketAddress apply(String value) { + return parse(value); + } + }; + + /** + * A function that uses {@link #getLocalAddress(int)} to map a local port number to an + * {@link InetSocketAddress}. + * If an {@link UnknownHostException} is thrown, it will be propagated as a + * {@link RuntimeException}. + */ + public static final Function<Integer, InetSocketAddress> INT_TO_INET = + new Function<Integer, InetSocketAddress>() { + @Override public InetSocketAddress apply(Integer port) { + try { + return getLocalAddress(port); + } catch (UnknownHostException e) { + throw Throwables.propagate(e); + } + } + }; + + public static final Function<InetSocketAddress, String> INET_TO_STR = + new Function<InetSocketAddress, String>() { + @Override public String apply(InetSocketAddress addr) { + return InetSocketAddressHelper.toString(addr); + } + }; + + /** + * Attempts to parse an endpoint spec into an InetSocketAddress. + * + * @param value the endpoint spec + * @return a parsed InetSocketAddress + * @throws NullPointerException if {@code value} is {@code null} + * @throws IllegalArgumentException if {@code value} cannot be parsed + */ + public static InetSocketAddress parse(String value) { + Preconditions.checkNotNull(value); + + String[] spec = value.split(":", 2); + if (spec.length != 2) { + throw new IllegalArgumentException("Invalid socket address spec: " + value); + } + + String host = spec[0]; + int port = asPort(spec[1]); + + return StringUtils.isEmpty(host) + ? new InetSocketAddress(port) + : InetSocketAddress.createUnresolved(host, port); + } + + /** + * Attempts to return a usable String given an InetSocketAddress. + * + * @param value the InetSocketAddress. + * @return the String representation of the InetSocketAddress. + */ + public static String toString(InetSocketAddress value) { + Preconditions.checkNotNull(value); + return value.getHostName() + ":" + value.getPort(); + } + + private static int asPort(String port) { + if ("*".equals(port)) { + return 0; + } + try { + return Integer.parseInt(port); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid port: " + port, e); + } + } + + public static InetSocketAddress getLocalAddress(int port) throws UnknownHostException { + String ipAddress = InetAddress.getLocalHost().getHostAddress(); + return new InetSocketAddress(ipAddress, port); + } + + private InetSocketAddressHelper() { + // utility + } + + /** + * Converts backend definitions (in host:port form) a set of socket addresses. + * + * @param backends Backends to convert. + * @return Sockets representing the provided backends. + */ + public static Set<InetSocketAddress> convertToSockets(Iterable<String> backends) { + return Sets.newHashSet(Iterables.transform(backends, STR_TO_INET)); + } +}
