Repository: incubator-htrace Updated Branches: refs/heads/master 97530fb91 -> 4b492b241
HTRACE-51 htraced java REST client (a.k.a java SpanReceiver for htraced) Project: http://git-wip-us.apache.org/repos/asf/incubator-htrace/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-htrace/commit/4b492b24 Tree: http://git-wip-us.apache.org/repos/asf/incubator-htrace/tree/4b492b24 Diff: http://git-wip-us.apache.org/repos/asf/incubator-htrace/diff/4b492b24 Branch: refs/heads/master Commit: 4b492b241e4709ebf0ce45b41f5a1417ac198bde Parents: 97530fb Author: stack <[email protected]> Authored: Fri Jan 30 14:54:36 2015 -0800 Committer: stack <[email protected]> Committed: Fri Jan 30 14:54:36 2015 -0800 ---------------------------------------------------------------------- htrace-core/pom.xml | 12 +- .../impl/HTracedRESTReceiver$PostSpans.class | Bin 0 -> 6048 bytes .../htrace/impl/HTracedRESTReceiver.class | Bin 0 -> 6342 bytes ...edRESTReceiver$TestHTraceConfiguration.class | Bin 0 -> 4215 bytes .../htrace/impl/TestHTracedRESTReceiver.class | Bin 0 -> 6253 bytes .../bin/org/apache/htrace/util/DataDir.class | Bin 0 -> 4054 bytes .../org/apache/htrace/util/HTracedProcess.class | Bin 0 -> 4217 bytes .../apache/htrace/util/TestHTracedProcess.class | Bin 0 -> 3779 bytes htrace-htraced/pom.xml | 150 +++++++++++ .../apache/htrace/impl/HTracedRESTReceiver.java | 251 +++++++++++++++++++ .../htrace/impl/TestHTracedRESTReceiver.java | 141 +++++++++++ .../java/org/apache/htrace/util/DataDir.java | 97 +++++++ .../org/apache/htrace/util/HTracedProcess.java | 103 ++++++++ .../apache/htrace/util/TestHTracedProcess.java | 93 +++++++ pom.xml | 21 +- 15 files changed, 856 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/4b492b24/htrace-core/pom.xml ---------------------------------------------------------------------- diff --git a/htrace-core/pom.xml b/htrace-core/pom.xml index 1d2f0ab..bbc3de8 100644 --- a/htrace-core/pom.xml +++ b/htrace-core/pom.xml @@ -138,20 +138,18 @@ language governing permissions and limitations under the License. --> <artifactId>junit</artifactId> <scope>test</scope> </dependency> - <!-- core specific deps. --> - <dependency> - <groupId>commons-logging</groupId> - <artifactId>commons-logging</artifactId> - </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-core</artifactId> - <version>2.4.0</version> </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> - <version>2.4.0</version> + </dependency> + <!-- core specific deps. --> + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/4b492b24/htrace-htraced/bin/org/apache/htrace/impl/HTracedRESTReceiver$PostSpans.class ---------------------------------------------------------------------- diff --git a/htrace-htraced/bin/org/apache/htrace/impl/HTracedRESTReceiver$PostSpans.class b/htrace-htraced/bin/org/apache/htrace/impl/HTracedRESTReceiver$PostSpans.class new file mode 100644 index 0000000..77d1559 Binary files /dev/null and b/htrace-htraced/bin/org/apache/htrace/impl/HTracedRESTReceiver$PostSpans.class differ http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/4b492b24/htrace-htraced/bin/org/apache/htrace/impl/HTracedRESTReceiver.class ---------------------------------------------------------------------- diff --git a/htrace-htraced/bin/org/apache/htrace/impl/HTracedRESTReceiver.class b/htrace-htraced/bin/org/apache/htrace/impl/HTracedRESTReceiver.class new file mode 100644 index 0000000..43b44b0 Binary files /dev/null and b/htrace-htraced/bin/org/apache/htrace/impl/HTracedRESTReceiver.class differ http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/4b492b24/htrace-htraced/bin/org/apache/htrace/impl/TestHTracedRESTReceiver$TestHTraceConfiguration.class ---------------------------------------------------------------------- diff --git a/htrace-htraced/bin/org/apache/htrace/impl/TestHTracedRESTReceiver$TestHTraceConfiguration.class b/htrace-htraced/bin/org/apache/htrace/impl/TestHTracedRESTReceiver$TestHTraceConfiguration.class new file mode 100644 index 0000000..8662503 Binary files /dev/null and b/htrace-htraced/bin/org/apache/htrace/impl/TestHTracedRESTReceiver$TestHTraceConfiguration.class differ http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/4b492b24/htrace-htraced/bin/org/apache/htrace/impl/TestHTracedRESTReceiver.class ---------------------------------------------------------------------- diff --git a/htrace-htraced/bin/org/apache/htrace/impl/TestHTracedRESTReceiver.class b/htrace-htraced/bin/org/apache/htrace/impl/TestHTracedRESTReceiver.class new file mode 100644 index 0000000..9456899 Binary files /dev/null and b/htrace-htraced/bin/org/apache/htrace/impl/TestHTracedRESTReceiver.class differ http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/4b492b24/htrace-htraced/bin/org/apache/htrace/util/DataDir.class ---------------------------------------------------------------------- diff --git a/htrace-htraced/bin/org/apache/htrace/util/DataDir.class b/htrace-htraced/bin/org/apache/htrace/util/DataDir.class new file mode 100644 index 0000000..ef80323 Binary files /dev/null and b/htrace-htraced/bin/org/apache/htrace/util/DataDir.class differ http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/4b492b24/htrace-htraced/bin/org/apache/htrace/util/HTracedProcess.class ---------------------------------------------------------------------- diff --git a/htrace-htraced/bin/org/apache/htrace/util/HTracedProcess.class b/htrace-htraced/bin/org/apache/htrace/util/HTracedProcess.class new file mode 100644 index 0000000..1de4f75 Binary files /dev/null and b/htrace-htraced/bin/org/apache/htrace/util/HTracedProcess.class differ http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/4b492b24/htrace-htraced/bin/org/apache/htrace/util/TestHTracedProcess.class ---------------------------------------------------------------------- diff --git a/htrace-htraced/bin/org/apache/htrace/util/TestHTracedProcess.class b/htrace-htraced/bin/org/apache/htrace/util/TestHTracedProcess.class new file mode 100644 index 0000000..95ec940 Binary files /dev/null and b/htrace-htraced/bin/org/apache/htrace/util/TestHTracedProcess.class differ http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/4b492b24/htrace-htraced/pom.xml ---------------------------------------------------------------------- diff --git a/htrace-htraced/pom.xml b/htrace-htraced/pom.xml new file mode 100644 index 0000000..6995ed4 --- /dev/null +++ b/htrace-htraced/pom.xml @@ -0,0 +1,150 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor +license agreements. See the NOTICE file distributed with this work for additional +information regarding copyright ownership. The ASF licenses this file to +You under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of +the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required +by applicable law or agreed to in writing, software distributed under the +License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +OF ANY KIND, either express or implied. See the License for the specific +language governing permissions and limitations under the License. --> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <artifactId>htrace-htraced</artifactId> + <packaging>jar</packaging> + + <parent> + <artifactId>htrace</artifactId> + <groupId>org.apache.htrace</groupId> + <version>3.2.0-incubating-SNAPSHOT</version> + <relativePath>..</relativePath> + </parent> + + <name>htrace-htraced</name> + <description>HTraced and HTraced clients</description> + <url>http://incubator.apache.org/projects/htrace.html</url> + + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + </properties> + + <build> + <plugins> + <plugin> + <!--Make it so assembly:single does nothing in here--> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <skipAssembly>true</skipAssembly> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-source-plugin</artifactId> + </plugin> + <plugin> + <artifactId>maven-javadoc-plugin</artifactId> + </plugin> + <plugin> + <artifactId>maven-compiler-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-gpg-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + </plugin> + <plugin> + <!-- explicitly define maven-deploy-plugin after other to force exec order --> + <artifactId>maven-deploy-plugin</artifactId> + </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <configuration> + <descriptorRefs> + <descriptorRef>jar-with-dependencies</descriptorRef> + </descriptorRefs> + </configuration> + </plugin> + <!--Move this to top-level. These shade patterns are common across components + --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <executions> + <execution> + <phase>package</phase> + <configuration> + <relocations> + <relocation> + <pattern>org.apache.commons.logging</pattern> + <shadedPattern>org.apache.htrace.commons.logging</shadedPattern> + </relocation> + <relocation> + <pattern>com.fasterxml.jackson</pattern> + <shadedPattern>org.apache.htrace.fasterxml.jackson</shadedPattern> + </relocation> + <relocation> + <pattern>org.eclipse.jetty</pattern> + <shadedPattern>org.apache.htrace.jetty</shadedPattern> + </relocation> + </relocations> + </configuration> + <goals> + <goal>shade</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + <dependencies> + <!-- Module deps. --> + <dependency> + <groupId>org.apache.htrace</groupId> + <artifactId>htrace-core</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.htrace</groupId> + <artifactId>htrace-core</artifactId> + <version>${project.version}</version> + <classifier>tests</classifier> + <scope>test</scope> + </dependency> + <!-- Global deps. --> + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + </dependency> + <!-- htraced rest client deps. --> + <!--Is this too much? Pulls down jetty-http, jetty-server, jetty-io + This is new-style jetty client, jetty9 and jdk7 required. + It can do async but we will use it synchronously at first. + Has nice tutorial: http://www.eclipse.org/jetty/documentation/9.1.5.v20140505/http-client-api.html + --> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-client</artifactId> + <version>9.2.6.v20141205</version> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/4b492b24/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedRESTReceiver.java ---------------------------------------------------------------------- diff --git a/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedRESTReceiver.java b/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedRESTReceiver.java new file mode 100644 index 0000000..38279f6 --- /dev/null +++ b/htrace-htraced/src/main/java/org/apache/htrace/impl/HTracedRESTReceiver.java @@ -0,0 +1,251 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.htrace.impl; + + +import java.io.IOException; +import java.net.URL; +import java.util.Queue; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.htrace.HTraceConfiguration; +import org.apache.htrace.Span; +import org.apache.htrace.SpanReceiver; +import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.client.api.Request; +import org.eclipse.jetty.client.util.StringContentProvider; +import org.eclipse.jetty.http.HttpField; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.http.HttpMethod; +import org.eclipse.jetty.http.HttpStatus; + +/** + * A {@link SpanReceiver} that passes Spans to htraced via REST. Implementation minimizes + * dependencies and aims for small footprint since this client will be the guest of another, + * the process traced. + * + * <p>Logs via commons-logging. Uses jetty client. Jetty has its own logging. To connect, see + * jetty logging to commons-logging and see https://issues.apache.org/jira/browse/HADOOP-6807 + * and http://blogs.bytecode.com.au/glen/2005/06/21/getting-your-logging-working-in-jetty.html. + * + * <p>This client depends on the REST defined in <code>rest.go</code> in the htraced REST server. + * + * <p>Create an instance by doing: + * <code>SpanReceiver receiver = new HTracedRESTReceiver(conf);</code> where conf is an instance + * of {@link HTraceConfiguration}. See the public keys defined below for what we will look for in + * the configuration. For example, set {@link #HTRACED_REST_URL_KEY} if + * <code>htraced</code> is in a non-standard location. Then call + * <code>receiver.receiveSpan(Span);</code> to send spans to an htraced + * instance. This method returns immediately. It sends the spans in background. + * + * <p>TODO: Shading works? + * TODO: Add lazy start; don't start background thread till a span gets queued. + * TODO: Add some metrics; how many times we've run, how many spans and what size we've sent. + */ +public class HTracedRESTReceiver implements SpanReceiver { + private static final Log LOG = LogFactory.getLog(HTracedRESTReceiver.class); + + // TODO: Take process name and add this to user agent? Would help debugging? + // @VisibleForTesting Protected so accessible from tests. + final HttpClient httpClient; + + /** + * REST URL to use writing Spans. + */ + private final String writeSpansRESTURL; + + /** + * Runs background task to do the REST PUT. + */ + private final ScheduledExecutorService scheduler; + + /** + * Keep around reference so can cancel on close any running scheduled task. + */ + private final ScheduledFuture<?> scheduledFuture; + + /** + * Timeout in milliseconds. + * For now, it is read and connect timeout. + */ + public static final String CLIENT_REST_TIMEOUT_MS_KEY = "client.rest.timeout.ms"; + private static final int CLIENT_REST_TIMEOUT_MS_DEFAULT = 60000; + + /** + * URL of the htraced REST server we are to talk to. + */ + public static final String HTRACED_REST_URL_KEY = "htraced.rest.url"; + private static final String HTRACED_REST_URL_DEFAULT = "http://localhost:9095/";; + + /** + * Maximum size of the queue to accumulate spans in. + * Cleared by the background thread that does the REST POST to htraced. + */ + public static final String CLIENT_REST_QUEUE_CAPACITY_KEY = "client.rest.queue.capacity"; + private static final int CLIENT_REST_QUEUE_CAPACITY_DEFAULT = 1000000; + + /** + * Period at which the background thread that does the REST POST to htraced in ms. + */ + public static final String CLIENT_REST_PERIOD_MS_KEY = "client.reset.period.ms"; + private static final int CLIENT_REST_PERIOD_MS_DEFAULT = 1000; + + /** + * Maximum spans to post to htraced at a time. + */ + public static final String CLIENT_REST_MAX_SPANS_AT_A_TIME_KEY = + "htrace.client.rest.batch.size"; + private static final int CLIENT_REST_MAX_SPANS_AT_A_TIME_DEFAULT = 100; + + /** + * Simple bounded queue to hold spans between periodic runs of the httpclient. + */ + private final Queue<Span> queue; + + /** + * Keep last time we logged we were at capacity; used to prevent flooding of logs with + * "at capacity" messages. + */ + private volatile long lastAtCapacityWarningLog = 0L; + + /** + * Constructor. + * You must call {@link #close()} post construction when done. + * @param conf + * @throws Exception + */ + public HTracedRESTReceiver(final HTraceConfiguration conf) throws Exception { + this.httpClient = new HttpClient(); + this.httpClient.setUserAgentField(new HttpField(HttpHeader.USER_AGENT, + this.getClass().getSimpleName())); + // Use same timeout for connection and idle for now. + int timeout = conf.getInt(CLIENT_REST_TIMEOUT_MS_KEY, CLIENT_REST_TIMEOUT_MS_DEFAULT); + this.httpClient.setConnectTimeout(timeout); + this.httpClient.setIdleTimeout(timeout); + int capacity = conf.getInt(CLIENT_REST_QUEUE_CAPACITY_KEY, CLIENT_REST_QUEUE_CAPACITY_DEFAULT); + this.queue = new ArrayBlockingQueue<Span>(capacity, true); + // Build up the writeSpans URL. + URL restServer = new URL(conf.get(HTRACED_REST_URL_KEY, HTRACED_REST_URL_DEFAULT)); + URL url = + new URL(restServer.getProtocol(), restServer.getHost(), restServer.getPort(), "/writeSpans"); + this.writeSpansRESTURL = url.toString(); + // Make a scheduler with one thread to run our POST of spans on a period. + this.scheduler = Executors.newScheduledThreadPool(1); + // Period at which we run the background thread that does the REST POST to htraced. + int periodInMs = conf.getInt(CLIENT_REST_PERIOD_MS_KEY, CLIENT_REST_PERIOD_MS_DEFAULT); + // Maximum spans to send in one go + int maxToSendAtATime = + conf.getInt(CLIENT_REST_MAX_SPANS_AT_A_TIME_KEY, CLIENT_REST_MAX_SPANS_AT_A_TIME_DEFAULT); + this.scheduledFuture = + this.scheduler.scheduleAtFixedRate(new PostSpans(this.queue, maxToSendAtATime), + periodInMs, periodInMs, TimeUnit.MILLISECONDS); + // Start up the httpclient. + this.httpClient.start(); + } + + /** + * POST spans runnable. + * Run on a period. Services the passed in queue taking spans and sending them to traced via http. + */ + private class PostSpans implements Runnable { + private final Queue<Span> q; + private final int maxToSendAtATime; + + private PostSpans(final Queue<Span> q, final int maxToSendAtATime) { + this.q = q; + this.maxToSendAtATime = maxToSendAtATime; + } + + @Override + public void run() { + Span span = null; + // Cycle until we drain queue. Send maxToSendAtATime if more than this in queue. + while ((span = this.q.poll()) != null) { + // We got a span. Send at least this one span. + Request request = httpClient.newRequest(writeSpansRESTURL).method(HttpMethod.POST); + request.header(HttpHeader.CONTENT_TYPE, "application/json"); + int count = 1; + request.content(new StringContentProvider(span.toJson())); + // Drain queue or until we have maxToSendAtATime spans, if more than just one. + while ((span = this.q.poll()) != null) { + request.content(new StringContentProvider(span.toJson())); + count++; + // If we've accumulated sufficient to send, go ahead and send what we have. Can do the + // rest in out next go around. + if (count > this.maxToSendAtATime) break; + } + try { + ContentResponse response = request.send(); + if (response.getStatus() == HttpStatus.OK_200) { + if (LOG.isDebugEnabled()) LOG.debug("POSTED " + count + " spans"); + } else { + LOG.error("Status: " + response.getStatus()); + LOG.error(response.getHeaders()); + LOG.error(response.getContentAsString()); + } + } catch (InterruptedException e) { + LOG.error(e); + } catch (TimeoutException e) { + LOG.error(e); + } catch (ExecutionException e) { + LOG.error(e); + } + } + } + } + + @Override + public void close() throws IOException { + if (this.scheduledFuture != null) this.scheduledFuture.cancel(true); + if (this.scheduler == null) this.scheduler.shutdown(); + if (this.httpClient != null) { + try { + this.httpClient.stop(); + } catch (Exception e) { + throw new IOException(e); + } + } + } + + // @VisibleForTesting + boolean isQueueEmpty() { + return this.queue.isEmpty(); + } + + @Override + public void receiveSpan(Span span) { + if (!this.queue.offer(span)) { + // TODO: If failed the offer, run the background thread now. I can't block though? + long now = System.nanoTime(); + // Only log every 5 minutes. Any more than this for a guest process is obnoxious + if ((now / 1000000) - lastAtCapacityWarningLog > 300000) { + LOG.warn("At capacity"); + this.lastAtCapacityWarningLog = now; + } + } + } +} http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/4b492b24/htrace-htraced/src/test/java/org/apache/htrace/impl/TestHTracedRESTReceiver.java ---------------------------------------------------------------------- diff --git a/htrace-htraced/src/test/java/org/apache/htrace/impl/TestHTracedRESTReceiver.java b/htrace-htraced/src/test/java/org/apache/htrace/impl/TestHTracedRESTReceiver.java new file mode 100644 index 0000000..fe9f1c0 --- /dev/null +++ b/htrace-htraced/src/test/java/org/apache/htrace/impl/TestHTracedRESTReceiver.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.htrace.impl; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.net.URL; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.htrace.HTraceConfiguration; +import org.apache.htrace.Span; +import org.apache.htrace.util.DataDir; +import org.apache.htrace.util.HTracedProcess; +import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.http.HttpStatus; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class TestHTracedRESTReceiver { + private static final Log LOG = LogFactory.getLog(TestHTracedRESTReceiver.class); + private URL restServerUrl;; + private DataDir dataDir; + HTracedProcess htraced; + + @Before + public void setUp() throws Exception { + this.dataDir = new DataDir(); + // Start on 9097. Would be better to start at port 0 and then ask server what port it managed + // to come up on. + this.restServerUrl = new URL("http://localhost:9097/";); + File tlDir = DataDir.getTopLevelOfCheckout(this.dataDir.getDataDir()); + File pathToHTracedBinary = HTracedProcess.getPathToHTraceBinaryFromTopLevel(tlDir); + this.htraced = new HTracedProcess(pathToHTracedBinary, dataDir.getDataDir(), restServerUrl); + } + + @After + public void tearDown() throws Exception { + if (this.htraced != null) this.htraced.destroy(); + } + + /** + * Our simple version of htrace configuration for testing. + */ + private final class TestHTraceConfiguration extends HTraceConfiguration { + private final URL restServerUrl; + + public TestHTraceConfiguration(final URL restServerUrl) { + this.restServerUrl = restServerUrl; + } + + @Override + public String get(String key) { + return null; + } + + @Override + public String get(String key, String defaultValue) { + if (key.equals(HTracedRESTReceiver.HTRACED_REST_URL_KEY)) { + return this.restServerUrl.toString(); + } + return defaultValue; + } + } + + /** + * Make sure the REST server basically works. + * @throws Exception + */ + @Test (timeout = 10000) + public void testBasicGet() throws Exception { + HTracedRESTReceiver receiver = + new HTracedRESTReceiver(new TestHTraceConfiguration(this.restServerUrl)); + try { + // Do basic a GET /server/info against localhost:9095 htraced + ContentResponse response = receiver.httpClient.GET(restServerUrl + "server/info"); + assertEquals("application/json", response.getMediaType()); + String content = processGET(response); + assertTrue(content.contains("ReleaseVersion")); + System.out.println(content); + } finally { + receiver.close(); + } + } + + private String processGET(final ContentResponse response) { + assertTrue("" + response.getStatus(), HttpStatus.OK_200 <= response.getStatus() && + response.getStatus() <= HttpStatus.NO_CONTENT_204); + return response.getContentAsString(); + } + + /** + * Send 100 spans then confirm they made it in. + * @throws Exception + */ + @Test (timeout = 10000) + public void testSendingSpans() throws Exception { + HTracedRESTReceiver receiver = + new HTracedRESTReceiver(new TestHTraceConfiguration(this.restServerUrl)); + try { + // TODO: Fix MilliSpan. Requires a parentid. Shouldn't have to have one else be explicit it + // is required. + for (int i = 0; i < 100; i++) { + Span span = new MilliSpan.Builder().parents(new long [] {1L}).spanId(i).build(); + LOG.info(span.toString()); + receiver.receiveSpan(span); + } + // Wait for the queue to empty before we go to check they made it over. + while (receiver.isQueueEmpty()) Thread.sleep(1); + // Read them all back. + for (int i = 0; i < 100; i++) { + // This is what the REST server expends when querying for a span id. + String findSpan = String.format("span/%016x", i); + ContentResponse response = receiver.httpClient.GET(restServerUrl + findSpan); + String content = processGET(response); + assertTrue(content != null && content.length() > 0); + LOG.info(content); + } + } finally { + receiver.close(); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/4b492b24/htrace-htraced/src/test/java/org/apache/htrace/util/DataDir.java ---------------------------------------------------------------------- diff --git a/htrace-htraced/src/test/java/org/apache/htrace/util/DataDir.java b/htrace-htraced/src/test/java/org/apache/htrace/util/DataDir.java new file mode 100644 index 0000000..74731fa --- /dev/null +++ b/htrace-htraced/src/test/java/org/apache/htrace/util/DataDir.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.htrace.util; + +import java.io.File; +import java.io.IOException; +import java.util.UUID; + +/** + * Small util for making a data directory for tests to use when running tests. We put it up at + * target/test-data/UUID. Create an instance of this class per unit test run and it will take + * care of setting up the dirs for you. Pass what is returned here as location from which to + * have daemons and tests dump data. + * TODO: Add close on exit. + */ +public class DataDir { + private File baseTestDir = null; + private File testDir = null; + + /** + * System property key to get base test directory value + */ + public static final String TEST_BASE_DIRECTORY_KEY = "test.data.base.dir"; + + /** + * Default base directory for test output. + */ + public static final String TEST_BASE_DIRECTORY_DEFAULT = "target"; + + public static final String TEST_BASE_DIRECTORY_NAME = "test-data"; + + /** + * @return Where to write test data on local filesystem; usually + * {@link #TEST_BASE_DIRECTORY_DEFAULT} + * Should not be used directly by the unit tests, hence its's private. + * Unit test will use a subdirectory of this directory. + * @see #setupDataTestDir() + */ + private synchronized File getBaseTestDir() { + if (this.baseTestDir != null) return this.baseTestDir; + String testHome = System.getProperty(TEST_BASE_DIRECTORY_KEY, TEST_BASE_DIRECTORY_DEFAULT); + this.baseTestDir = new File(testHome, TEST_BASE_DIRECTORY_NAME); + return this.baseTestDir; + } + + /** + * @return Absolute path to the dir created by this instance. + * @throws IOException + */ + public synchronized File getDataDir() throws IOException { + if (this.testDir != null) return this.testDir; + this.testDir = new File(getBaseTestDir(), UUID.randomUUID().toString()); + if (!this.testDir.exists()) { + if (!this.testDir.mkdirs()) throw new IOException("Failed mkdirs for " + this.testDir); + } + // Return absolute path. A relative passed to htraced will have it create data dirs relative + // to its data dir rather than in it. + return this.testDir.getAbsoluteFile(); + } + + /** + * Fragile. Ugly. Presumes paths. Best we can do for now until htraced comes local to this module + * and is moved out of src dir. + * @param dataDir A datadir gotten from {@link #getDataDir()} + * @return Top-level of the checkout. + */ + public static File getTopLevelOfCheckout(final File dataDir) { + // Need absolute else we run out of road when dir is relative to this module. + File absolute = dataDir.getAbsoluteFile(); + // Check we are where we think we are. + File testDataDir = absolute.getParentFile(); + if (!testDataDir.getName().equals(TEST_BASE_DIRECTORY_NAME)) { + throw new IllegalArgumentException(dataDir.toString()); + } + // Do another check. + File targetDir = testDataDir.getParentFile(); + if (!targetDir.getName().equals(TEST_BASE_DIRECTORY_DEFAULT)) { + throw new IllegalArgumentException(dataDir.toString()); + } + // Back up last two dirs out of the htrace-htraced dir. + return targetDir.getParentFile().getParentFile(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/4b492b24/htrace-htraced/src/test/java/org/apache/htrace/util/HTracedProcess.java ---------------------------------------------------------------------- diff --git a/htrace-htraced/src/test/java/org/apache/htrace/util/HTracedProcess.java b/htrace-htraced/src/test/java/org/apache/htrace/util/HTracedProcess.java new file mode 100644 index 0000000..12343f7 --- /dev/null +++ b/htrace-htraced/src/test/java/org/apache/htrace/util/HTracedProcess.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.htrace.util; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.lang.ProcessBuilder.Redirect; +import java.net.URL; + +/** + * To get instance of HTraced up and running, create an instance of this class. + * Upon successful construction, htraced is running using <code>dataDir</code> as directory to + * host data (leveldbs and logs). + * TODO: We expect to find the htraced in a very particular place. Fragile. Will break if stuff + * moves. + * TODO: What if a port clash? How to have it come up another port then ask the process what port + * it is running on? + */ +public class HTracedProcess extends Process { + private final Process delegate; + + public HTracedProcess(final File pathToHTracedBinary, final File dataDir, final URL url) + throws IOException { + // web.address for htraced is hostname ':' port; no 'scheme' yet. + String webAddress = url.getHost() + ":" + url.getPort(); + // Pass cmdline args to htraced to it uses our test dir for data. + ProcessBuilder pb = new ProcessBuilder(pathToHTracedBinary.toString(), + " -Dlog.level=TRACE", + "-Dweb.address=" + webAddress, + "-Ddata.store.clear=true", + "-Ddata.store.directories=" + dataDir.toString()); + pb.redirectErrorStream(true); + // Inherit STDERR/STDOUT i/o; dumps on console for now. Can add logs later. + pb.inheritIO(); + pb.directory(dataDir); + this.delegate = pb.start(); + assert pb.redirectInput() == Redirect.PIPE; + assert pb.redirectOutput().file() == dataDir; + assert this.delegate.getInputStream().read() == -1; + } + + public int hashCode() { + return delegate.hashCode(); + } + + public OutputStream getOutputStream() { + throw new UnsupportedOperationException("Unsupported until complaint; output on STDOUT"); + } + + public InputStream getInputStream() { + throw new UnsupportedOperationException("Unsupported until complaint; output on STDOUT"); + } + + public boolean equals(Object obj) { + return delegate.equals(obj); + } + + public InputStream getErrorStream() { + throw new UnsupportedOperationException("Unsupported until complaint; output on STDOUT"); + } + + public int waitFor() throws InterruptedException { + return delegate.waitFor(); + } + + public int exitValue() { + return delegate.exitValue(); + } + + public void destroy() { + delegate.destroy(); + } + + public String toString() { + return delegate.toString(); + } + + /** + * Ugly but how else to do file-math? + * @param topLevel Presumes top-level of the htrace checkout. + * @return Path to the htraced binary. + */ + public static File getPathToHTraceBinaryFromTopLevel(final File topLevel) { + return new File(new File(new File(new File(new File(topLevel, "htrace-core"), "src"), "go"), + "build"), "htraced"); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/4b492b24/htrace-htraced/src/test/java/org/apache/htrace/util/TestHTracedProcess.java ---------------------------------------------------------------------- diff --git a/htrace-htraced/src/test/java/org/apache/htrace/util/TestHTracedProcess.java b/htrace-htraced/src/test/java/org/apache/htrace/util/TestHTracedProcess.java new file mode 100644 index 0000000..38f90e5 --- /dev/null +++ b/htrace-htraced/src/test/java/org/apache/htrace/util/TestHTracedProcess.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.htrace.util; + +import static org.junit.Assert.assertTrue; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.URL; +import java.net.URLConnection; + +import org.junit.Before; +import org.junit.Test; + +/** + * Test putting up an htraced and making sure it basically works. + * Makes presumption about paths; where data is relative to the htraced binary, etc., encoded + * in methods in the below. + */ +public class TestHTracedProcess { + private DataDir testDir = null; + private final int TIMEOUT = 10000; + + @Before + public void setupTest() { + this.testDir = new DataDir(); + } + + /* + * Do a basic GET of the server info from the running htraced instance. + */ + private String doGet(final URL url) throws IOException { + URLConnection connection = url.openConnection(); + connection.setConnectTimeout(TIMEOUT); + connection.setReadTimeout(TIMEOUT); + connection.connect(); + StringBuffer sb = new StringBuffer(); + BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream())); + try { + String line = null; + while ((line = reader.readLine()) != null) { + System.out.println(line); + sb.append(line); + } + } finally { + reader.close(); + } + return sb.toString(); + } + + /** + * Put up an htraced instance and do a Get against /server/info. + * @throws IOException + * @throws InterruptedException + */ + @Test (timeout=10000) + public void testStartStopHTraced() throws IOException, InterruptedException { + // TODO: Make the test port random so no classes if concurrent test runs. Anything better + // I can do here? Pass a zero and have the daemon tell me where it is successfully listening? + String restURL = "http://localhost:9096/";; + URL restServerURL = new URL(restURL); + HTracedProcess htraced = null; + File dataDir = this.testDir.getDataDir(); + File topLevel = DataDir.getTopLevelOfCheckout(dataDir); + try { + htraced = new HTracedProcess(HTracedProcess.getPathToHTraceBinaryFromTopLevel(topLevel), + dataDir, restServerURL); + String str = doGet(new URL(restServerURL + "server/info")); + // Assert we go something back. + assertTrue(str.contains("ReleaseVersion")); + // Assert that the datadir is not empty. + } finally { + if (htraced != null) htraced.destroy(); + System.out.println("ExitValue=" + htraced.exitValue()); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/4b492b24/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 8d8d505..8bf899a 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,7 @@ language governing permissions and limitations under the License. --> <module>htrace-zipkin</module> <module>htrace-hbase</module> <module>htrace-flume</module> + <module>htrace-htraced</module> </modules> <licenses> @@ -212,16 +213,16 @@ language governing permissions and limitations under the License. --> </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>2.1</version> + </plugin> </plugins> </pluginManagement> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <version>2.1</version> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-gpg-plugin</artifactId> </plugin> <plugin> @@ -296,6 +297,16 @@ language governing permissions and limitations under the License. --> <version>4.10</version> <scope>test</scope> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-core</artifactId> + <version>2.4.0</version> + </dependency> + <dependency> + <groupId>com.fasterxml.jackson.core</groupId> + <artifactId>jackson-databind</artifactId> + <version>2.4.0</version> + </dependency> </dependencies> </dependencyManagement> <distributionManagement>
