Repository: phoenix Updated Branches: refs/heads/master 9185f760b -> b7f46c105
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/phoenix-hadoop-compat/src/main/java/org/apache/phoenix/trace/TestableMetricsWriter.java ---------------------------------------------------------------------- diff --git a/phoenix-hadoop-compat/src/main/java/org/apache/phoenix/trace/TestableMetricsWriter.java b/phoenix-hadoop-compat/src/main/java/org/apache/phoenix/trace/TestableMetricsWriter.java new file mode 100644 index 0000000..b6bc75d --- /dev/null +++ b/phoenix-hadoop-compat/src/main/java/org/apache/phoenix/trace/TestableMetricsWriter.java @@ -0,0 +1,30 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.trace; + +import org.apache.phoenix.metrics.MetricsWriter; + +/** + * Marker interface for a MetricsWriter that can be registered to the current metrics system. The + * writer should convert from the metrics information it receives from the metrics system to Phoenix + * records that the MetricsWriter can read (and subsequently write). + */ +public interface TestableMetricsWriter { + + public void setWriterForTesting(MetricsWriter writer); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/phoenix-hadoop-compat/src/main/java/org/apache/phoenix/trace/TracingCompat.java ---------------------------------------------------------------------- diff --git a/phoenix-hadoop-compat/src/main/java/org/apache/phoenix/trace/TracingCompat.java b/phoenix-hadoop-compat/src/main/java/org/apache/phoenix/trace/TracingCompat.java new file mode 100644 index 0000000..6ec12de --- /dev/null +++ b/phoenix-hadoop-compat/src/main/java/org/apache/phoenix/trace/TracingCompat.java @@ -0,0 +1,94 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.trace; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hbase.CompatibilityFactory; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.Pair; +import org.apache.phoenix.metrics.MetricsWriter; +import org.cloudera.htrace.Span; +import org.cloudera.htrace.SpanReceiver; + +/** + * Utilities for tracing that are common among the compatibility and core classes. + */ +public class TracingCompat { + + private static final Log LOG = LogFactory.getLog(TracingCompat.class); + + /** + * @return a new SpanReceiver that will write to the correct metrics system + */ + public static SpanReceiver newTraceMetricSource() { + return CompatibilityFactory.getInstance(PhoenixSpanReceiver.class); + } + + public static final String DEFAULT_STATS_TABLE_NAME = "PHOENIX.TRACING_STATS"; + + /** + * Configuration key to overwrite the tablename that should be used as the target table + */ + public static final String TARGET_TABLE_CONF_KEY = + "org.apache.phoenix._internal.trace.tablename"; + + public static final String METRIC_SOURCE_KEY = "phoenix."; + + /** Set context to enable filtering */ + public static final String METRICS_CONTEXT = "tracing"; + + public static void addAnnotation(Span span, String message, int value) { + span.addKVAnnotation(message.getBytes(), Bytes.toBytes(value)); + } + + public static Pair<String, String> readAnnotation(byte[] key, byte[] value) { + return new Pair<String, String>(new String(key), Integer.toString(Bytes.toInt(value))); + } + + public static MetricsWriter initializeWriter(String clazz) { + try { + MetricsWriter writer = + Class.forName(clazz).asSubclass(MetricsWriter.class).newInstance(); + writer.initialize(); + return writer; + } catch (InstantiationException e) { + LOG.error("Failed to create metrics writer: " + clazz, e); + } catch (IllegalAccessException e) { + LOG.error("Failed to create metrics writer: " + clazz, e); + } catch (ClassNotFoundException e) { + LOG.error("Failed to create metrics writer: " + clazz, e); + } + return null; + } + + /** + * @see #getTraceMetricName(String) + */ + public static final String getTraceMetricName(long traceId) { + return getTraceMetricName(Long.toString(traceId)); + } + + /** + * @param traceId unique id of the trace + * @return the name of the metric record that should be generated for a given trace + */ + public static final String getTraceMetricName(String traceId) { + return METRIC_SOURCE_KEY + traceId; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/phoenix-hadoop-compat/src/test/java/org/apache/phoenix/metrics/LoggingSink.java ---------------------------------------------------------------------- diff --git a/phoenix-hadoop-compat/src/test/java/org/apache/phoenix/metrics/LoggingSink.java b/phoenix-hadoop-compat/src/test/java/org/apache/phoenix/metrics/LoggingSink.java new file mode 100644 index 0000000..97682b3 --- /dev/null +++ b/phoenix-hadoop-compat/src/test/java/org/apache/phoenix/metrics/LoggingSink.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.metrics; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.phoenix.trace.TracingCompat; + +/** + * Simple sink that just logs the output of all the metrics that start with + * {@link TracingCompat#METRIC_SOURCE_KEY} + */ +public class LoggingSink implements MetricsWriter { + + private static final Log LOG = LogFactory.getLog(LoggingSink.class); + + @Override + public void initialize() { + } + + @Override + public void addMetrics(PhoenixMetricsRecord record) { + // we could wait until flush, but this is a really lightweight process, so we just write + // them + // as soon as we get them + if (!LOG.isDebugEnabled()) { + return; + } + LOG.debug("Found record:" + record.name()); + for (PhoenixAbstractMetric metric : record.metrics()) { + // just print the metric we care about + if (metric.getName().startsWith(TracingCompat.METRIC_SOURCE_KEY)) { + LOG.debug("\t metric:" + metric); + } + } + } + + @Override + public void flush() { + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/phoenix-hadoop-compat/src/test/java/org/apache/phoenix/metrics/TracingTestCompat.java ---------------------------------------------------------------------- diff --git a/phoenix-hadoop-compat/src/test/java/org/apache/phoenix/metrics/TracingTestCompat.java b/phoenix-hadoop-compat/src/test/java/org/apache/phoenix/metrics/TracingTestCompat.java new file mode 100644 index 0000000..8dd8a41 --- /dev/null +++ b/phoenix-hadoop-compat/src/test/java/org/apache/phoenix/metrics/TracingTestCompat.java @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.metrics; + +import org.apache.hadoop.hbase.CompatibilityFactory; +import org.apache.phoenix.trace.TestableMetricsWriter; + +/** + * Utility class for testing tracing + */ +public class TracingTestCompat { + + private TracingTestCompat() { + assert false; + } + + public static TestableMetricsWriter newTraceMetricSink() { + return CompatibilityFactory.getInstance(TestableMetricsWriter.class); + } + + /** + * Register the sink with the metrics system, so we don't need to specify it in the conf + * @param sink + */ + public static void registerSink(MetricsWriter sink) { + TestableMetricsWriter writer = newTraceMetricSink(); + writer.setWriterForTesting(sink); + Metrics.getManager().register("phoenix", "test sink gets logged", writer); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/phoenix-hadoop2-compat/pom.xml ---------------------------------------------------------------------- diff --git a/phoenix-hadoop2-compat/pom.xml b/phoenix-hadoop2-compat/pom.xml index 2781bcf..7944a51 100644 --- a/phoenix-hadoop2-compat/pom.xml +++ b/phoenix-hadoop2-compat/pom.xml @@ -29,4 +29,49 @@ </parent> <artifactId>phoenix-hadoop2-compat</artifactId> <name>Phoenix Hadoop2 Compatibility</name> -</project> + + <dependencies> + <!-- Intra-project dependencies --> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hadoop-compat</artifactId> + </dependency> + <!-- HBase --> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-common</artifactId> + <exclusions> + <exclusion> + <artifactId>hadoop-core</artifactId> + <groupId>org.apache.hadoop</groupId> + </exclusion> + </exclusions> + </dependency> + <!-- Hadoop --> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${hadoop-two.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-annotations</artifactId> + <version>${hadoop-two.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-common</artifactId> + <version>${hadoop-two.version}</version> + </dependency> + <!-- Other --> + <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </dependency> + <!-- Test --> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-all</artifactId> + </dependency> + </dependencies> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/phoenix-hadoop2-compat/src/main/java/org/apache/phoenix/metrics/MetricsManagerImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-hadoop2-compat/src/main/java/org/apache/phoenix/metrics/MetricsManagerImpl.java b/phoenix-hadoop2-compat/src/main/java/org/apache/phoenix/metrics/MetricsManagerImpl.java new file mode 100644 index 0000000..03e06a5 --- /dev/null +++ b/phoenix-hadoop2-compat/src/main/java/org/apache/phoenix/metrics/MetricsManagerImpl.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more contributor license + * agreements. See the NOTICE file distributed with this work for additional information regarding + * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. You may obtain a + * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable + * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" + * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License + * for the specific language governing permissions and limitations under the License. + */ +package org.apache.phoenix.metrics; + +import java.util.Arrays; + +import org.apache.hadoop.metrics2.MetricsSink; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.MetricsSystem; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; + +import com.google.common.base.Preconditions; + +/** + * + */ +public class MetricsManagerImpl implements MetricsManager { + + private MetricsSystem system; + + @Override + /** + * Register a metrics sink + * @param <T> the type of the sink. + * @param sink to register + * @param name of the sink. Must be unique. + * @param desc the description of the sink + * @return the sink + * @throws IllegalArgumentException if sink is not a MetricsSink + */ + public <T> T register(String name, String desc, T sink) { + isA(sink, MetricsSink.class); + return (T) system.register(name, desc, (MetricsSink) sink); + } + + public <T> T registerSource(String name, String desc, T source) { + isA(source, MetricsSource.class); + return (T) system.register(name, desc, (MetricsSource) source); + } + + @Override + public void initialize(String prefix) { + this.system = DefaultMetricsSystem.initialize(prefix); + } + + private <T> void isA(T object, Class<?>... classes) { + boolean match = false; + for (Class<?> clazz : classes) { + if (clazz.isAssignableFrom(object.getClass())) { + match = true; + break; + } + } + Preconditions.checkArgument(match, object + " is not one of " + Arrays.toString(classes)); + } + + @Override + public void shutdown() { + if (this.system != null) { + this.system.shutdown(); + } + } +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/phoenix-hadoop2-compat/src/main/java/org/apache/phoenix/trace/MetricsInfoImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-hadoop2-compat/src/main/java/org/apache/phoenix/trace/MetricsInfoImpl.java b/phoenix-hadoop2-compat/src/main/java/org/apache/phoenix/trace/MetricsInfoImpl.java new file mode 100644 index 0000000..47c1dda --- /dev/null +++ b/phoenix-hadoop2-compat/src/main/java/org/apache/phoenix/trace/MetricsInfoImpl.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.trace; + +import com.google.common.base.Objects; +import static com.google.common.base.Preconditions.*; +import org.apache.hadoop.metrics2.MetricsInfo; + +/** + * Making implementing metric info a little easier + * <p> + * Just a copy of the same from Hadoop, but exposed for usage. + */ +public class MetricsInfoImpl implements MetricsInfo { + private final String name, description; + + MetricsInfoImpl(String name, String description) { + this.name = checkNotNull(name, "name"); + this.description = checkNotNull(description, "description"); + } + + @Override public String name() { + return name; + } + + @Override public String description() { + return description; + } + + @Override public boolean equals(Object obj) { + if (obj instanceof MetricsInfo) { + MetricsInfo other = (MetricsInfo) obj; + return Objects.equal(name, other.name()) && + Objects.equal(description, other.description()); + } + return false; + } + + @Override public int hashCode() { + return Objects.hashCode(name, description); + } + + @Override public String toString() { + return Objects.toStringHelper(this) + .add("name", name).add("description", description) + .toString(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/phoenix-hadoop2-compat/src/main/java/org/apache/phoenix/trace/PhoenixMetricsWriter.java ---------------------------------------------------------------------- diff --git a/phoenix-hadoop2-compat/src/main/java/org/apache/phoenix/trace/PhoenixMetricsWriter.java b/phoenix-hadoop2-compat/src/main/java/org/apache/phoenix/trace/PhoenixMetricsWriter.java new file mode 100644 index 0000000..03230ee --- /dev/null +++ b/phoenix-hadoop2-compat/src/main/java/org/apache/phoenix/trace/PhoenixMetricsWriter.java @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.trace; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; + +import javax.annotation.Nullable; + +import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.SubsetConfiguration; +import org.apache.hadoop.metrics2.AbstractMetric; +import org.apache.hadoop.metrics2.MetricsRecord; +import org.apache.hadoop.metrics2.MetricsSink; +import org.apache.hadoop.metrics2.MetricsTag; +import org.apache.phoenix.metrics.MetricsWriter; +import org.apache.phoenix.metrics.PhoenixAbstractMetric; +import org.apache.phoenix.metrics.PhoenixMetricTag; +import org.apache.phoenix.metrics.PhoenixMetricsRecord; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Function; +import com.google.common.base.Preconditions; +import com.google.common.collect.Iterators; + +/** + * Translate metrics from a Hadoop2 metrics2 metric to a generic PhoenixMetric that a + * {@link MetricsWriter} can then write out. + * <p> + * This class becomes unnecessary once we drop Hadoop1 support. + */ +public class PhoenixMetricsWriter implements MetricsSink, TestableMetricsWriter { + + /** + * Metrics configuration key for the class that should be used for writing the output + */ + public static final String PHOENIX_METRICS_WRITER_CLASS = "phoenix.sink.writer-class"; + + public static void setWriterClass(MetricsWriter writer, Configuration conf) { + conf.setProperty(PHOENIX_METRICS_WRITER_CLASS, writer.getClass().getName()); + } + + private MetricsWriter writer; + + @Override + public void init(SubsetConfiguration config) { + // instantiate the configured writer class + String clazz = config.getString(PHOENIX_METRICS_WRITER_CLASS); + this.writer = TracingCompat.initializeWriter(clazz); + Preconditions.checkNotNull(writer, "Could not correctly initialize metrics writer!"); + } + + @Override + @VisibleForTesting + public void setWriterForTesting(MetricsWriter writer) { + this.writer = writer; + } + + @Override + public void putMetrics(MetricsRecord record) { + writer.addMetrics(wrap(record)); + } + + @Override + public void flush() { + writer.flush(); + } + + /** + * Convert the passed record to a {@link PhoenixMetricsRecord} + * @param record to convert + * @return a generic {@link PhoenixMetricsRecord} that delegates to the record in all things + */ + private PhoenixMetricsRecord wrap(final MetricsRecord record) { + return new PhoenixMetricsRecord() { + + @Override + public String name() { + return record.name(); + } + + @Override + public String description() { + return record.description(); + } + + @Override + public Iterable<PhoenixAbstractMetric> metrics() { + final Iterable<AbstractMetric> iterable = record.metrics(); + return new Iterable<PhoenixAbstractMetric>(){ + + @Override + public Iterator<PhoenixAbstractMetric> iterator() { + final Iterator<AbstractMetric> iter = iterable.iterator(); + return Iterators.transform(iter, new Function<AbstractMetric, PhoenixAbstractMetric>() { + + @Override + @Nullable + public PhoenixAbstractMetric apply(@Nullable final AbstractMetric input) { + if (input == null) { + return null; + } + return new PhoenixAbstractMetric() { + + @Override + public Number value() { + return input.value(); + } + + @Override + public String getName() { + return input.name(); + } + + @Override + public String toString() { + return input.toString(); + } + }; + } + }); + } + }; + } + + @Override + public Collection<PhoenixMetricTag> tags() { + Collection<PhoenixMetricTag> tags = new ArrayList<PhoenixMetricTag>(); + Collection<MetricsTag> origTags = record.tags(); + for (final MetricsTag tag : origTags) { + tags.add(new PhoenixMetricTag() { + + @Override + public String name() { + return tag.name(); + } + + @Override + public String description() { + return tag.description(); + } + + @Override + public String value() { + return tag.value(); + } + + @Override + public String toString() { + return tag.toString(); + } + + }); + } + return tags; + } + + }; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/phoenix-hadoop2-compat/src/main/java/org/apache/phoenix/trace/TraceMetricSource.java ---------------------------------------------------------------------- diff --git a/phoenix-hadoop2-compat/src/main/java/org/apache/phoenix/trace/TraceMetricSource.java b/phoenix-hadoop2-compat/src/main/java/org/apache/phoenix/trace/TraceMetricSource.java new file mode 100644 index 0000000..5876771 --- /dev/null +++ b/phoenix-hadoop2-compat/src/main/java/org/apache/phoenix/trace/TraceMetricSource.java @@ -0,0 +1,192 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.trace; + +import static org.apache.phoenix.metrics.MetricInfo.ANNOTATION; +import static org.apache.phoenix.metrics.MetricInfo.TAG; +import static org.apache.phoenix.metrics.MetricInfo.END; +import static org.apache.phoenix.metrics.MetricInfo.PARENT; +import static org.apache.phoenix.metrics.MetricInfo.SPAN; +import static org.apache.phoenix.metrics.MetricInfo.START; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.hadoop.hbase.util.Pair; +import org.apache.hadoop.metrics2.MetricsCollector; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecordBuilder; +import org.apache.hadoop.metrics2.MetricsSource; +import org.apache.hadoop.metrics2.MetricsTag; +import org.apache.hadoop.metrics2.lib.Interns; +import org.apache.phoenix.metrics.MetricInfo; +import org.apache.phoenix.metrics.Metrics; +import org.apache.phoenix.metrics.MetricsManager; +import org.cloudera.htrace.HTraceConfiguration; +import org.cloudera.htrace.Span; +import org.cloudera.htrace.SpanReceiver; +import org.cloudera.htrace.TimelineAnnotation; +import org.cloudera.htrace.impl.MilliSpan; + +/** + * Sink for request traces ({@link SpanReceiver}) that pushes writes to {@link MetricsSource} in a + * format that we can more easily consume. + * <p> + * <p> + * Rather than write directly to a phoenix table, we drop it into the metrics queue so we can more + * cleanly handle it asyncrhonously.Currently, {@link MilliSpan} submits the span in a synchronized + * block to all the receivers, which could have a lot of overhead if we are submitting to multiple + * receivers. + * <p> + * The format of the generated metrics is this: + * <ol> + * <li>All Metrics from the same span have the same name (allowing correlation in the sink)</li> + * <li>The description of the metric describes what it contains. For instance, + * <ul> + * <li>{@link MetricInfo#PARENT} is the id of the parent of this span. (Root span is + * {@link Span#ROOT_SPAN_ID}).</li> + * <li>{@value MetricInfo#START} is the start time of the span</li> + * <li>{@value MetricInfo#END} is the end time of the span</li> + * </ul></li> + * <li>Each span's messages are contained in a {@link MetricsTag} with the same name as above and a + * generic counter for the number of messages (to differentiate messages and provide timeline + * ordering).</li> + * </ol> + * <p> + * <i>So why even submit to metrics2 framework if we only have a single source?</i> + * <p> + * This allows us to make the updates in batches. We might have spans that finish before other spans + * (for instance in the same parent). By batching the updates we can lessen the overhead on the + * client, which is also busy doing 'real' work. <br> + * We could make our own queue and manage batching and filtering and dropping extra metrics, but + * that starts to get complicated fast (its not as easy as it sounds) so we use metrics2 to abstract + * out that pipeline and also provides us flexibility to dump metrics to other sources. + * <p> + * This is a somewhat rough implementation - we do excessive locking for correctness, + * rather than trying to make it fast, for the moment. + */ +public class TraceMetricSource implements PhoenixSpanReceiver, MetricsSource { + + private static final String EMPTY_STRING = ""; + + /** This must match the prefix that we are using in the hadoop-metrics2 config */ + private static final String METRICS_SYSTEM_NAME = "phoenix"; + private static final String CONTEXT = "tracing"; + + private List<Metric> spans = new ArrayList<Metric>(); + + public TraceMetricSource() { + MetricsManager manager = Metrics.getManager(); + manager.initialize(METRICS_SYSTEM_NAME); + + // Register this instance. + // For right now, we ignore the MBean registration issues that show up in DEBUG logs. Basically, + // we need a Jmx MBean compliant name. We'll get to a better name when we want that later + manager.registerSource(CONTEXT, "Phoenix call tracing", this); + } + + @Override + public void receiveSpan(Span span) { + Metric builder = new Metric(span); + // add all the metrics for the span + builder.addCounter(Interns.info(SPAN.traceName, EMPTY_STRING), span.getSpanId()); + builder.addCounter(Interns.info(PARENT.traceName, EMPTY_STRING), span.getParentId()); + builder.addCounter(Interns.info(START.traceName, EMPTY_STRING), span.getStartTimeMillis()); + builder.addCounter(Interns.info(END.traceName, EMPTY_STRING), span.getStopTimeMillis()); + // add the tags to the span. They were written in order received so we mark them as such + for (TimelineAnnotation ta : span.getTimelineAnnotations()) { + builder.add(new MetricsTag(Interns.info(TAG.traceName, Long.toString(ta.getTime())), ta + .getMessage())); + } + + // add the annotations. We assume they are serialized as strings and integers, but that can + // change in the future + Map<byte[], byte[]> annotations = span.getKVAnnotations(); + for (Entry<byte[], byte[]> annotation : annotations.entrySet()) { + Pair<String, String> val = + TracingCompat.readAnnotation(annotation.getKey(), annotation.getValue()); + builder.add(new MetricsTag(Interns.info(ANNOTATION.traceName, val.getFirst()), val + .getSecond())); + } + + // add the span to the list we care about + synchronized (this) { + spans.add(builder); + } + } + + @Override + public void getMetrics(MetricsCollector collector, boolean all) { + synchronized (this) { + for (Metric span : spans) { + MetricsRecordBuilder builder = collector.addRecord(new MetricsInfoImpl(TracingCompat + .getTraceMetricName(span.id), span.desc)); + builder.setContext(TracingCompat.METRICS_CONTEXT); + for (Pair<MetricsInfo, Long> metric : span.counters) { + builder.addCounter(metric.getFirst(), metric.getSecond()); + } + for (MetricsTag tag : span.tags) { + builder.add(tag); + } + } + // reset the spans so we don't keep a big chunk of memory around + spans = new ArrayList<Metric>(); + } + } + + @Override + public void close() throws IOException { + // noop + } + + @Override + public void configure(HTraceConfiguration conf) { + // noop + } + + private class Metric { + + List<Pair<MetricsInfo, Long>> counters = new ArrayList<Pair<MetricsInfo, Long>>(); + List<MetricsTag> tags = new ArrayList<MetricsTag>(); + private String id; + private String desc; + + public Metric(Span span) { + this.id = Long.toString(span.getTraceId()); + this.desc = span.getDescription(); + } + + /** + * @param metricsInfoImpl + * @param startTimeMillis + */ + public void addCounter(MetricsInfo metricsInfoImpl, long startTimeMillis) { + counters.add(new Pair<MetricsInfo, Long>(metricsInfoImpl, startTimeMillis)); + } + + /** + * @param metricsTag + */ + public void add(MetricsTag metricsTag) { + tags.add(metricsTag); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/phoenix-hadoop2-compat/src/main/resources/META-INF/services/org.apache.phoenix.metrics.MetricsManager ---------------------------------------------------------------------- diff --git a/phoenix-hadoop2-compat/src/main/resources/META-INF/services/org.apache.phoenix.metrics.MetricsManager b/phoenix-hadoop2-compat/src/main/resources/META-INF/services/org.apache.phoenix.metrics.MetricsManager new file mode 100644 index 0000000..8430a48 --- /dev/null +++ b/phoenix-hadoop2-compat/src/main/resources/META-INF/services/org.apache.phoenix.metrics.MetricsManager @@ -0,0 +1 @@ +org.apache.phoenix.metrics.MetricsManagerImpl \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/phoenix-hadoop2-compat/src/main/resources/META-INF/services/org.apache.phoenix.trace.PhoenixSpanReceiver ---------------------------------------------------------------------- diff --git a/phoenix-hadoop2-compat/src/main/resources/META-INF/services/org.apache.phoenix.trace.PhoenixSpanReceiver b/phoenix-hadoop2-compat/src/main/resources/META-INF/services/org.apache.phoenix.trace.PhoenixSpanReceiver new file mode 100644 index 0000000..3694093 --- /dev/null +++ b/phoenix-hadoop2-compat/src/main/resources/META-INF/services/org.apache.phoenix.trace.PhoenixSpanReceiver @@ -0,0 +1 @@ +org.apache.phoenix.trace.TraceMetricSource \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/phoenix-hadoop2-compat/src/main/resources/META-INF/services/org.apache.phoenix.trace.TestableMetricsWriter ---------------------------------------------------------------------- diff --git a/phoenix-hadoop2-compat/src/main/resources/META-INF/services/org.apache.phoenix.trace.TestableMetricsWriter b/phoenix-hadoop2-compat/src/main/resources/META-INF/services/org.apache.phoenix.trace.TestableMetricsWriter new file mode 100644 index 0000000..b10af39 --- /dev/null +++ b/phoenix-hadoop2-compat/src/main/resources/META-INF/services/org.apache.phoenix.trace.TestableMetricsWriter @@ -0,0 +1 @@ +org.apache.phoenix.trace.PhoenixMetricsWriter \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/phoenix-hadoop2-compat/src/test/java/org/apache/hadoop/metrics2/impl/ExposedMetricCounterLong.java ---------------------------------------------------------------------- diff --git a/phoenix-hadoop2-compat/src/test/java/org/apache/hadoop/metrics2/impl/ExposedMetricCounterLong.java b/phoenix-hadoop2-compat/src/test/java/org/apache/hadoop/metrics2/impl/ExposedMetricCounterLong.java new file mode 100644 index 0000000..33ca738 --- /dev/null +++ b/phoenix-hadoop2-compat/src/test/java/org/apache/hadoop/metrics2/impl/ExposedMetricCounterLong.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.metrics2.impl; + +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.impl.MetricCounterLong; + +/** + * + */ +public class ExposedMetricCounterLong extends MetricCounterLong { + + /** + * @param info + * @param value + */ + public ExposedMetricCounterLong(MetricsInfo info, long value) { + super(info, value); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/phoenix-hadoop2-compat/src/test/java/org/apache/hadoop/metrics2/impl/ExposedMetricsRecordImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-hadoop2-compat/src/test/java/org/apache/hadoop/metrics2/impl/ExposedMetricsRecordImpl.java b/phoenix-hadoop2-compat/src/test/java/org/apache/hadoop/metrics2/impl/ExposedMetricsRecordImpl.java new file mode 100644 index 0000000..bcb8b43 --- /dev/null +++ b/phoenix-hadoop2-compat/src/test/java/org/apache/hadoop/metrics2/impl/ExposedMetricsRecordImpl.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.metrics2.impl; + +import java.util.List; + +import org.apache.hadoop.metrics2.AbstractMetric; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsTag; +import org.apache.hadoop.metrics2.impl.MetricsRecordImpl; + +/** + * Helper class to access the package-private {@link MetricsRecordImpl} + */ +@SuppressWarnings("javadoc") +public class ExposedMetricsRecordImpl extends MetricsRecordImpl { + + /** + * @param info + * @param timestamp + * @param tags + * @param metrics + */ + public ExposedMetricsRecordImpl(MetricsInfo info, long timestamp, List<MetricsTag> tags, + Iterable<AbstractMetric> metrics) { + super(info, timestamp, tags, metrics); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/phoenix-hadoop2-compat/src/test/java/org/apache/hadoop/metrics2/lib/ExposedMetricsInfoImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-hadoop2-compat/src/test/java/org/apache/hadoop/metrics2/lib/ExposedMetricsInfoImpl.java b/phoenix-hadoop2-compat/src/test/java/org/apache/hadoop/metrics2/lib/ExposedMetricsInfoImpl.java new file mode 100644 index 0000000..6daf604 --- /dev/null +++ b/phoenix-hadoop2-compat/src/test/java/org/apache/hadoop/metrics2/lib/ExposedMetricsInfoImpl.java @@ -0,0 +1,32 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.metrics2.lib; + +/** + * Helper class to expose access to the {@link MetricsInfoImpl} + */ +public class ExposedMetricsInfoImpl extends MetricsInfoImpl { + + /** + * @param name + * @param description + */ + public ExposedMetricsInfoImpl(String name, String description) { + super(name, description); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/phoenix-hadoop2-compat/src/test/java/org/apache/phoenix/trace/PhoenixMetricsWriterTest.java ---------------------------------------------------------------------- diff --git a/phoenix-hadoop2-compat/src/test/java/org/apache/phoenix/trace/PhoenixMetricsWriterTest.java b/phoenix-hadoop2-compat/src/test/java/org/apache/phoenix/trace/PhoenixMetricsWriterTest.java new file mode 100644 index 0000000..0c8bbab --- /dev/null +++ b/phoenix-hadoop2-compat/src/test/java/org/apache/phoenix/trace/PhoenixMetricsWriterTest.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.trace; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.List; + +import org.apache.hadoop.metrics2.AbstractMetric; +import org.apache.hadoop.metrics2.MetricsInfo; +import org.apache.hadoop.metrics2.MetricsRecord; +import org.apache.hadoop.metrics2.MetricsTag; +import org.apache.hadoop.metrics2.impl.ExposedMetricCounterLong; +import org.apache.hadoop.metrics2.impl.ExposedMetricsRecordImpl; +import org.apache.hadoop.metrics2.lib.ExposedMetricsInfoImpl; +import org.apache.phoenix.metrics.MetricInfo; +import org.apache.phoenix.metrics.MetricsWriter; +import org.apache.phoenix.metrics.PhoenixAbstractMetric; +import org.apache.phoenix.metrics.PhoenixMetricTag; +import org.apache.phoenix.metrics.PhoenixMetricsRecord; +import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import com.google.common.collect.Lists; + +/** + * Test that we correctly convert between hadoop2 metrics2 and generic phoenix metrics. + */ +public class PhoenixMetricsWriterTest { + + @Test + public void testTranslation() throws Exception { + // hook up a sink we can test + MetricsWriter mockSink = Mockito.mock(MetricsWriter.class); + + // writer that will translate to the sink (specific to hadoop version used) + PhoenixMetricsWriter writer = new PhoenixMetricsWriter(); + writer.setWriterForTesting(mockSink); + + // create a simple metrics record + final long traceid = 987654; + MetricsInfo info = new ExposedMetricsInfoImpl(TracingCompat.getTraceMetricName(traceid), + "Some generic trace"); + // setup some metrics for the span + long spanid = 10; + AbstractMetric span = new ExposedMetricCounterLong(new ExposedMetricsInfoImpl( + MetricInfo.SPAN.traceName, ""), spanid); + long parentid = 11; + AbstractMetric parent = new ExposedMetricCounterLong(new ExposedMetricsInfoImpl( + MetricInfo.PARENT.traceName, ""), parentid); + long startTime = 12; + AbstractMetric start = new ExposedMetricCounterLong(new ExposedMetricsInfoImpl( + MetricInfo.START.traceName, ""), startTime); + long endTime = 13; + AbstractMetric end = new ExposedMetricCounterLong(new ExposedMetricsInfoImpl( + MetricInfo.END.traceName, ""), endTime); + final List<AbstractMetric> metrics = Lists.newArrayList(span, parent, start, end); + + // create an annotation as well + String annotation = "test annotation for a span"; + MetricsTag tag = new MetricsTag( + new ExposedMetricsInfoImpl(MetricInfo.ANNOTATION.traceName, "0"), annotation); + String hostnameValue = "host-name.value"; + MetricsTag hostname = new MetricsTag(new ExposedMetricsInfoImpl(MetricInfo.HOSTNAME.traceName, + ""), hostnameValue); + final List<MetricsTag> tags = Lists.newArrayList(hostname, tag); + + MetricsRecord record = new ExposedMetricsRecordImpl(info, System.currentTimeMillis(), tags, + metrics); + + // setup the mocking/validation for the sink + Mockito.doAnswer(new Answer<Void>() { + + @Override + public Void answer(InvocationOnMock invocation) throws Throwable { + PhoenixMetricsRecord record = (PhoenixMetricsRecord) invocation.getArguments()[0]; + //validate that we got the right fields in the record + assertEquals("phoenix.987654", record.name()); + assertEquals("Some generic trace", record.description()); + int count = 0; + for (PhoenixAbstractMetric metric : record.metrics()) { + count++; + //find the matching metric in the list + boolean found = false; + for(AbstractMetric expected : metrics){ + if(expected.name().equals(metric.getName())){ + found = true; + // make sure the rest of the info matches + assertEquals("Metric value mismatch", expected.value(), metric.value()); + } + } + assertTrue("Didn't find an expected metric to match "+metric, found); + } + assertEquals("Number of metrics is received is wrong", metrics.size(), count); + + count = 0; + for (PhoenixMetricTag tag : record.tags()) { + count++; + // find the matching metric in the list + boolean found = false; + for (MetricsTag expected : tags) { + if (expected.name().equals(tag.name())) { + found = true; + // make sure the rest of the info matches + assertEquals("Tag value mismatch", expected.value(), tag.value()); + assertEquals("Tag description mismatch", expected.description(), tag.description()); + } + } + assertTrue("Didn't find an expected metric to match " + tag, found); + } + assertEquals("Number of tags is received is wrong", tags.size(), count); + return null; + } + + }).when(mockSink).addMetrics(Mockito.any(PhoenixMetricsRecord.class)); + + // actually do the update + writer.putMetrics(record); + writer.flush(); + + Mockito.verify(mockSink).addMetrics(Mockito.any(PhoenixMetricsRecord.class)); + Mockito.verify(mockSink).flush(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/phoenix-hadoop2-compat/src/test/java/org/apache/phoenix/trace/TracingTest.java ---------------------------------------------------------------------- diff --git a/phoenix-hadoop2-compat/src/test/java/org/apache/phoenix/trace/TracingTest.java b/phoenix-hadoop2-compat/src/test/java/org/apache/phoenix/trace/TracingTest.java new file mode 100644 index 0000000..ffe6c82 --- /dev/null +++ b/phoenix-hadoop2-compat/src/test/java/org/apache/phoenix/trace/TracingTest.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.trace; + +import static org.junit.Assert.assertNotNull; + +import org.junit.Test; + +public class TracingTest { + + /** + * Test that we can correctly load a class that will convert the tracing output to metrics + * @throws Exception on failure + */ + @Test + public void testLoadTracingToMetrics() throws Exception{ + assertNotNull("Didn't find a trace receiver", TracingCompat.newTraceMetricSource()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/b7f46c10/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index b51d115..180d9da 100644 --- a/pom.xml +++ b/pom.xml @@ -559,7 +559,8 @@ <id>hadoop-1</id> <activation> <property> - <name>!hadoop.profile</name> + <name>hadoop.profile</name> + <value>1</value> </property> </activation> <modules> @@ -633,6 +634,11 @@ </exclusion> </exclusions> </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-hadoop-compat</artifactId> + <version>${hbase-hadoop1.version}</version> + </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-test</artifactId> @@ -649,8 +655,7 @@ <id>hadoop-2</id> <activation> <property> - <name>hadoop.profile</name> - <value>2</value> + <name>!hadoop.profile</name> </property> </activation> <modules> @@ -670,7 +675,7 @@ <version>${project.version}</version> </dependency> - <!-- Hadoop dependencies --> + <!-- HBase dependencies --> <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-testing-util</artifactId> @@ -689,6 +694,11 @@ </dependency> <dependency> <groupId>org.apache.hbase</groupId> + <artifactId>hbase-common</artifactId> + <version>${hbase-hadoop2.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> <artifactId>hbase-protocol</artifactId> <version>${hbase-hadoop2.version}</version> </dependency> @@ -698,24 +708,38 @@ <version>${hbase-hadoop2.version}</version> </dependency> <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-hadoop-compat</artifactId> + <version>${hbase-hadoop2.version}</version> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-hadoop-compat</artifactId> + <version>${hbase-hadoop2.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + + <!-- Hadoop Dependencies --> + <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - <version>${hadoop.version}</version> + <version>${hadoop-two.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-annotations</artifactId> - <version>${hadoop.version}</version> + <version>${hadoop-two.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> - <version>${hadoop.version}</version> + <version>${hadoop-two.version}</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-minicluster</artifactId> - <version>${hadoop.version}</version> + <version>${hadoop-two.version}</version> <optional>true</optional> <scope>test</scope> </dependency>
