adding colin review fixes on storing all parents
Project: http://git-wip-us.apache.org/repos/asf/incubator-htrace/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-htrace/commit/641d05bf Tree: http://git-wip-us.apache.org/repos/asf/incubator-htrace/tree/641d05bf Diff: http://git-wip-us.apache.org/repos/asf/incubator-htrace/diff/641d05bf Branch: refs/heads/master Commit: 641d05bfe3b65e4df7c9097d7f3a3c8f76179f9b Parents: 4b2e213 Author: Nisala Nirmana <[email protected]> Authored: Mon Aug 29 01:41:56 2016 +0530 Committer: Nisala Nirmana <[email protected]> Committed: Mon Aug 29 01:41:56 2016 +0530 ---------------------------------------------------------------------- .../htrace/impl/KuduReceiverConstants.java | 7 ++- .../apache/htrace/impl/KuduSpanReceiver.java | 41 +++++++++---- .../htrace/impl/TestKuduSpanReceiver.java | 61 +++++++++++++++----- .../apache/htrace/impl/TestKuduSpanViewer.java | 6 -- 4 files changed, 83 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/641d05bf/htrace-kudu/src/main/java/org/apache/htrace/impl/KuduReceiverConstants.java ---------------------------------------------------------------------- diff --git a/htrace-kudu/src/main/java/org/apache/htrace/impl/KuduReceiverConstants.java b/htrace-kudu/src/main/java/org/apache/htrace/impl/KuduReceiverConstants.java index 805ec80..c7f7484 100644 --- a/htrace-kudu/src/main/java/org/apache/htrace/impl/KuduReceiverConstants.java +++ b/htrace-kudu/src/main/java/org/apache/htrace/impl/KuduReceiverConstants.java @@ -25,14 +25,17 @@ public class KuduReceiverConstants { static final String DEFAULT_KUDU_MASTER_PORT = "7051"; static final String KUDU_SPAN_TABLE_KEY = "kudu.span.table"; static final String DEFAULT_KUDU_SPAN_TABLE = "span"; + static final String KUDU_SPAN_PARENT_TABLE_KEY = "kudu.span.parent.table"; + static final String DEFAULT_KUDU_SPAN_PARENT_TABLE = "span.parent"; static final String KUDU_SPAN_TIMELINE_ANNOTATION_TABLE_KEY = "kudu.span.timeline.annotation.table"; static final String DEFAULT_KUDU_SPAN_TIMELINE_ANNOTATION_TABLE = "span.timeline"; static final String DEFAULT_KUDU_COLUMN_SPAN_TRACE_ID = "trace_id"; static final String DEFAULT_KUDU_COLUMN_SPAN_START_TIME = "start_time"; static final String DEFAULT_KUDU_COLUMN_SPAN_STOP_TIME = "stop_time"; static final String DEFAULT_KUDU_COLUMN_SPAN_SPAN_ID = "span_id"; - static final String DEFAULT_KUDU_COLUMN_SPAN_PARENT_ID_LOW = "parent_id_low"; - static final String DEFAULT_KUDU_COLUMN_SPAN_PARENT_ID_HIGH = "parent_id_high"; + static final String DEFAULT_KUDU_COLUMN_PARENT_ID_LOW = "parent_id_low"; + static final String DEFAULT_KUDU_COLUMN_PARENT_ID_HIGH = "parent_id_high"; + static final String DEFAULT_KUDU_COLUMN_PARENT_CHILD_SPANID = "parent_child_span_id"; static final String DEFAULT_KUDU_COLUMN_SPAN_DESCRIPTION = "description"; static final String DEFAULT_KUDU_COLUMN_SPAN_PARENT = "parent"; static final String DEFAULT_KUDU_COLUMN_TIMELINE_TIME = "time"; http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/641d05bf/htrace-kudu/src/main/java/org/apache/htrace/impl/KuduSpanReceiver.java ---------------------------------------------------------------------- diff --git a/htrace-kudu/src/main/java/org/apache/htrace/impl/KuduSpanReceiver.java b/htrace-kudu/src/main/java/org/apache/htrace/impl/KuduSpanReceiver.java index 745f24d..5db017b 100644 --- a/htrace-kudu/src/main/java/org/apache/htrace/impl/KuduSpanReceiver.java +++ b/htrace-kudu/src/main/java/org/apache/htrace/impl/KuduSpanReceiver.java @@ -44,8 +44,6 @@ public class KuduSpanReceiver extends SpanReceiver { private String column_span_start_time; private String column_span_stop_time; private String column_span_span_id; - private String column_span_parent_id_low; - private String column_span_parent_id_high; private String column_span_description; private String column_span_parent; @@ -55,6 +53,15 @@ public class KuduSpanReceiver extends SpanReceiver { private String column_timeline_message; private String column_timeline_span_id; + private String table_span_parent; + private String column_parent_id_low; + private String column_parent_id_high; + private String column_parent_child_span_id; + + private KuduTable tableSpan; + private KuduTable tableTimeline; + private KuduTable tableParent; + public KuduSpanReceiver(HTraceConfiguration conf) { String masterHost; @@ -114,13 +121,18 @@ public class KuduSpanReceiver extends SpanReceiver { this.table_span = conf.get(KuduReceiverConstants.KUDU_SPAN_TABLE_KEY, KuduReceiverConstants.DEFAULT_KUDU_SPAN_TABLE); this.table_timeline = conf.get(KuduReceiverConstants.KUDU_SPAN_TIMELINE_ANNOTATION_TABLE_KEY, KuduReceiverConstants.DEFAULT_KUDU_SPAN_TIMELINE_ANNOTATION_TABLE); + this.table_span_parent = conf.get(KuduReceiverConstants.KUDU_SPAN_PARENT_TABLE_KEY, + KuduReceiverConstants.DEFAULT_KUDU_SPAN_PARENT_TABLE); //default column names have used this.column_span_trace_id = KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_TRACE_ID; this.column_span_start_time = KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_START_TIME; this.column_span_stop_time = KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_STOP_TIME; this.column_span_span_id = KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_SPAN_ID; - this.column_span_parent_id_low = KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_PARENT_ID_LOW; - this.column_span_parent_id_high = KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_PARENT_ID_HIGH; + + this.column_parent_id_low = KuduReceiverConstants.DEFAULT_KUDU_COLUMN_PARENT_ID_LOW; + this.column_parent_id_high = KuduReceiverConstants.DEFAULT_KUDU_COLUMN_PARENT_ID_HIGH; + this.column_parent_child_span_id = KuduReceiverConstants.DEFAULT_KUDU_COLUMN_PARENT_CHILD_SPANID; + this.column_span_description = KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_DESCRIPTION; this.column_span_parent = KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_PARENT; this.column_timeline_time = KuduReceiverConstants.DEFAULT_KUDU_COLUMN_TIMELINE_TIME; @@ -134,6 +146,13 @@ public class KuduSpanReceiver extends SpanReceiver { } session = client.newSession(); } + try { + tableSpan = client.openTable(table_span); + tableTimeline = client.openTable(table_timeline); + tableParent = client.openTable(table_span_parent); + } catch (java.lang.Exception ex) { + LOG.warn("Failed to open kudu tables to store Spans. " + ex.getMessage()); + } } @Override @@ -153,7 +172,6 @@ public class KuduSpanReceiver extends SpanReceiver { @Override public void receiveSpan(Span span) { try { - KuduTable tableSpan = client.openTable(table_span); Insert spanInsert = tableSpan.newInsert(); PartialRow spanRow = spanInsert.getRow(); spanRow.addLong(column_span_trace_id, span.getSpanId().getLow()); @@ -161,12 +179,16 @@ public class KuduSpanReceiver extends SpanReceiver { spanRow.addLong(column_span_stop_time, span.getStopTimeMillis()); spanRow.addLong(column_span_span_id, span.getSpanId().getHigh()); if (span.getParents().length == 0) { - spanRow.addLong(column_span_parent_id_low, 0); - spanRow.addLong(column_span_parent_id_high, 0); spanRow.addBoolean(column_span_parent, true); } else if (span.getParents().length > 0) { - spanRow.addLong(column_span_parent_id_low, span.getParents()[0].getLow()); - spanRow.addLong(column_span_parent_id_high, span.getParents()[0].getHigh()); + for (int i = 0; i < span.getParents().length; i++) { + Insert parentInsert = tableParent.newInsert(); + PartialRow parentRow = parentInsert.getRow(); + parentRow.addLong(column_parent_id_low, span.getParents()[i].getLow()); + parentRow.addLong(column_parent_id_high, span.getParents()[i].getHigh()); + parentRow.addLong(column_parent_child_span_id, span.getSpanId().getLow()); + session.apply(parentInsert); + } spanRow.addBoolean(column_span_parent, false); } spanRow.addString(column_span_description, span.getDescription()); @@ -174,7 +196,6 @@ public class KuduSpanReceiver extends SpanReceiver { long annotationCounter = 0; for (TimelineAnnotation ta : span.getTimelineAnnotations()) { annotationCounter++; - KuduTable tableTimeline = client.openTable(table_timeline); Insert timelineInsert = tableTimeline.newInsert(); PartialRow timelineRow = timelineInsert.getRow(); timelineRow.addLong(column_timeline_timeline_id, span.getSpanId().getLow() + annotationCounter); http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/641d05bf/htrace-kudu/src/test/java/org/apache/htrace/impl/TestKuduSpanReceiver.java ---------------------------------------------------------------------- diff --git a/htrace-kudu/src/test/java/org/apache/htrace/impl/TestKuduSpanReceiver.java b/htrace-kudu/src/test/java/org/apache/htrace/impl/TestKuduSpanReceiver.java index c13970d..4ac3657 100644 --- a/htrace-kudu/src/test/java/org/apache/htrace/impl/TestKuduSpanReceiver.java +++ b/htrace-kudu/src/test/java/org/apache/htrace/impl/TestKuduSpanReceiver.java @@ -83,12 +83,6 @@ public class TestKuduSpanReceiver extends BaseKuduTest { span_columns.add(new ColumnSchema.ColumnSchemaBuilder(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_SPAN_ID, Type.INT64) .build()); - span_columns.add(new ColumnSchema.ColumnSchemaBuilder(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_PARENT_ID_HIGH, - Type.INT64) - .build()); - span_columns.add(new ColumnSchema.ColumnSchemaBuilder(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_PARENT_ID_LOW, - Type.INT64) - .build()); span_columns.add(new ColumnSchema.ColumnSchemaBuilder(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_PARENT, Type.BOOL) .build()); @@ -121,6 +115,25 @@ public class TestKuduSpanReceiver extends BaseKuduTest { Schema timelineSchema = new Schema(timeline_columns); client.createTable(KuduReceiverConstants.DEFAULT_KUDU_SPAN_TIMELINE_ANNOTATION_TABLE, timelineSchema, new CreateTableOptions().setRangePartitionColumns(rangeKeysTimeline)); + + List<ColumnSchema> parent_columns = new ArrayList(); + parent_columns.add(new ColumnSchema.ColumnSchemaBuilder + (KuduReceiverConstants.DEFAULT_KUDU_COLUMN_PARENT_ID_LOW, Type.INT64) + .key(true) + .build()); + parent_columns.add(new ColumnSchema.ColumnSchemaBuilder(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_PARENT_ID_HIGH, + Type.INT64) + .build()); + parent_columns.add(new ColumnSchema.ColumnSchemaBuilder(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_PARENT_CHILD_SPANID, + Type.INT64) + .build()); + List<String> rangeKeysParent= new ArrayList<>(); + rangeKeysParent.add(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_PARENT_ID_LOW); + Schema parentSchema = new Schema(parent_columns); + client.createTable(KuduReceiverConstants.DEFAULT_KUDU_SPAN_PARENT_TABLE, parentSchema, + new CreateTableOptions().setRangePartitionColumns(rangeKeysParent)); + + } @Ignore @@ -139,6 +152,9 @@ public class TestKuduSpanReceiver extends BaseKuduTest { TraceScope scope = tracer.newScope("testKuduScope"); scope.addTimelineAnnotation("test"); Span testSpan = scope.getSpan(); + TraceScope childScope = tracer.newScope("testKuduChildScope", new SpanId(100,200)); + Span childScopeSpan = childScope.getSpan(); + childScope.close(); scope.close(); tracer.close(); KuduClient client = BaseKuduTest.syncClient; @@ -148,8 +164,6 @@ public class TestKuduSpanReceiver extends BaseKuduTest { spanColumns.add(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_DESCRIPTION); spanColumns.add(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_START_TIME); spanColumns.add(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_STOP_TIME); - spanColumns.add(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_PARENT_ID_HIGH); - spanColumns.add(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_PARENT_ID_LOW); KuduScanner scanner = client.newScannerBuilder(client.openTable(KuduReceiverConstants.DEFAULT_KUDU_SPAN_TABLE)) .setProjectedColumnNames(spanColumns) .build(); @@ -165,13 +179,32 @@ public class TestKuduSpanReceiver extends BaseKuduTest { .description(result.getString(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_DESCRIPTION)) .begin(result.getLong(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_START_TIME)) .end(result.getLong(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_STOP_TIME)); - if (!(result.getLong(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_PARENT_ID_HIGH) == 0 && - result.getLong(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_PARENT_ID_LOW) == 0)) { - SpanId[] parents = new SpanId[1]; - parents[0] = new SpanId(result.getLong(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_PARENT_ID_HIGH), - result.getLong(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_PARENT_ID_LOW)); - builder.parents(parents); + + List<String> parentColumns = new ArrayList<>(); + parentColumns.add(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_PARENT_ID_LOW); + parentColumns.add(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_PARENT_ID_HIGH); + parentColumns.add(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_PARENT_CHILD_SPANID); + KuduScanner parentScanner = client + .newScannerBuilder(client.openTable(KuduReceiverConstants.DEFAULT_KUDU_SPAN_PARENT_TABLE)) + .setProjectedColumnNames(parentColumns) + .addPredicate(KuduPredicate + .newComparisonPredicate(new ColumnSchema.ColumnSchemaBuilder + (KuduReceiverConstants.DEFAULT_KUDU_COLUMN_PARENT_CHILD_SPANID, Type.INT64) + .build(), KuduPredicate.ComparisonOp.EQUAL, traceId)) + .build(); + List<SpanId> parentList = new LinkedList<SpanId>(); + while (parentScanner.hasMoreRows()) { + RowResultIterator parentResults = parentScanner.nextRows(); + while (parentResults.hasNext()) { + RowResult parentRow = parentResults.next(); + parentList.add(new SpanId( + parentRow.getLong(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_PARENT_ID_HIGH), + parentRow.getLong(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_PARENT_ID_LOW))); + } } + SpanId[] array = new SpanId[parentList.size()]; + parentList.toArray(array); + builder.parents(array); List<String> timelineColumns = new ArrayList<>(); timelineColumns.add(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_TIMELINE_TIME); timelineColumns.add(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_TIMELINE_MESSAGE); http://git-wip-us.apache.org/repos/asf/incubator-htrace/blob/641d05bf/htrace-kudu/src/test/java/org/apache/htrace/impl/TestKuduSpanViewer.java ---------------------------------------------------------------------- diff --git a/htrace-kudu/src/test/java/org/apache/htrace/impl/TestKuduSpanViewer.java b/htrace-kudu/src/test/java/org/apache/htrace/impl/TestKuduSpanViewer.java index 7dd2807..c32a3bd 100644 --- a/htrace-kudu/src/test/java/org/apache/htrace/impl/TestKuduSpanViewer.java +++ b/htrace-kudu/src/test/java/org/apache/htrace/impl/TestKuduSpanViewer.java @@ -76,12 +76,6 @@ public class TestKuduSpanViewer extends BaseKuduTest { span_columns.add(new ColumnSchema.ColumnSchemaBuilder(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_SPAN_ID, Type.INT64) .build()); - span_columns.add(new ColumnSchema.ColumnSchemaBuilder(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_PARENT_ID_LOW, - Type.INT64) - .build()); - span_columns.add(new ColumnSchema.ColumnSchemaBuilder(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_PARENT_ID_HIGH, - Type.INT64) - .build()); span_columns.add(new ColumnSchema.ColumnSchemaBuilder(KuduReceiverConstants.DEFAULT_KUDU_COLUMN_SPAN_PARENT, Type.BOOL) .build());
