Repository: phoenix Updated Branches: refs/heads/master 93f560575 -> 0440aca51
PHOENIX-1115 Provide a SQL command to turn tracing on/off(Rajeshbabu) Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/0440aca5 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/0440aca5 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/0440aca5 Branch: refs/heads/master Commit: 0440aca51916d689f3ea94d4ef708d73887f7994 Parents: 93f5605 Author: Rajeshbabu Chintaguntla <rajeshb...@apache.org> Authored: Sat Feb 28 05:49:53 2015 +0530 Committer: Rajeshbabu Chintaguntla <rajeshb...@apache.org> Committed: Sat Feb 28 05:49:53 2015 +0530 ---------------------------------------------------------------------- .../phoenix/trace/PhoenixTracingEndToEndIT.java | 32 ++- phoenix-core/src/main/antlr3/PhoenixSQL.g | 10 + .../apache/phoenix/compile/TraceQueryPlan.java | 220 +++++++++++++++++++ .../apache/phoenix/jdbc/PhoenixConnection.java | 17 ++ .../apache/phoenix/jdbc/PhoenixStatement.java | 24 +- .../apache/phoenix/parse/ParseNodeFactory.java | 4 + .../apache/phoenix/parse/TraceStatement.java | 43 ++++ .../org/apache/phoenix/trace/util/Tracing.java | 10 + 8 files changed, 358 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/0440aca5/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java b/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java index 201de38..53d22c5 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java @@ -19,12 +19,15 @@ package org.apache.phoenix.trace; import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import java.sql.Connection; +import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.util.Collection; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -33,6 +36,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.metrics2.MetricsSource; import org.apache.phoenix.coprocessor.BaseScannerRegionObserver; +import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.metrics.Metrics; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.trace.TraceReader.SpanInfo; @@ -391,7 +395,33 @@ public class PhoenixTracingEndToEndIT extends BaseTracingTestIT { assertAnnotationPresent(TENANT_ID_ATTRIB, tenantId, conn); // CurrentSCN is also added as an annotation. Not tested here because it screws up test setup. } - + + @Test + public void testTraceOnOrOff() throws Exception { + Connection conn1 = DriverManager.getConnection(getUrl()); + try{ + Statement statement = conn1.createStatement(); + ResultSet rs = statement.executeQuery("TRACE ON"); + assertTrue(rs.next()); + long traceId = ((PhoenixConnection) conn1).getTraceScope().getSpan() + .getTraceId(); + assertEquals(rs.getLong(1), traceId); + assertEquals(rs.getLong("trace_id"), traceId); + assertFalse(rs.next()); + + rs = statement.executeQuery("TRACE OFF"); + assertTrue(rs.next()); + assertEquals(rs.getLong(1), traceId); + assertEquals(rs.getLong("trace_id"), traceId); + assertFalse(rs.next()); + + rs = statement.executeQuery("TRACE OFF"); + assertFalse(rs.next()); + } finally { + conn1.close(); + } + } + private void assertAnnotationPresent(final String annotationKey, final String annotationValue, Connection conn) throws Exception { boolean tracingComplete = checkStoredTraces(conn, new TraceChecker(){ @Override http://git-wip-us.apache.org/repos/asf/phoenix/blob/0440aca5/phoenix-core/src/main/antlr3/PhoenixSQL.g ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/antlr3/PhoenixSQL.g b/phoenix-core/src/main/antlr3/PhoenixSQL.g index cda93fe..9e843a0 100644 --- a/phoenix-core/src/main/antlr3/PhoenixSQL.g +++ b/phoenix-core/src/main/antlr3/PhoenixSQL.g @@ -33,6 +33,7 @@ tokens AS='as'; OUTER='outer'; ON='on'; + OFF='off'; IN='in'; GROUP='group'; HAVING='having'; @@ -107,6 +108,7 @@ tokens UPDATE='update'; STATISTICS='statistics'; COLUMNS='columns'; + TRACE='trace'; } @@ -160,6 +162,7 @@ import org.apache.phoenix.schema.types.PUnsignedTime; import org.apache.phoenix.schema.types.PUnsignedTimestamp; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.parse.LikeParseNode.LikeType; +import org.apache.phoenix.trace.util.Tracing; } @lexer::header { @@ -365,6 +368,7 @@ non_select_node returns [BindableStatement ret] | s=drop_index_node | s=alter_index_node | s=alter_table_node + | s=trace_node | s=create_sequence_node | s=drop_sequence_node | s=update_statistics_node @@ -498,6 +502,12 @@ alter_index_node returns [AlterIndexStatement ret] {ret = factory.alterIndex(factory.namedTable(null, TableName.create(t.getSchemaName(), i.getName())), t.getTableName(), ex!=null, PIndexState.valueOf(SchemaUtil.normalizeIdentifier(s.getText()))); } ; +// Parse a trace statement. +trace_node returns [TraceStatement ret] + : TRACE (flag = ON| flag = OFF) + {ret = factory.trace(Tracing.isTraceOn(flag.getText()));} + ; + // Parse an alter table statement. alter_table_node returns [AlterTableStatement ret] : ALTER (TABLE | v=VIEW) t=from_table_name http://git-wip-us.apache.org/repos/asf/phoenix/blob/0440aca5/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java new file mode 100644 index 0000000..9eb5877 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TraceQueryPlan.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.compile; + +import java.sql.ParameterMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.phoenix.compile.GroupByCompiler.GroupBy; +import org.apache.phoenix.compile.OrderByCompiler.OrderBy; +import org.apache.phoenix.expression.Determinism; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.expression.LiteralExpression; +import org.apache.phoenix.expression.RowKeyColumnExpression; +import org.apache.phoenix.iterate.ResultIterator; +import org.apache.phoenix.jdbc.PhoenixConnection; +import org.apache.phoenix.jdbc.PhoenixParameterMetaData; +import org.apache.phoenix.jdbc.PhoenixStatement; +import org.apache.phoenix.metrics.MetricInfo; +import org.apache.phoenix.parse.FilterableStatement; +import org.apache.phoenix.parse.LiteralParseNode; +import org.apache.phoenix.parse.ParseNodeFactory; +import org.apache.phoenix.parse.TraceStatement; +import org.apache.phoenix.query.KeyRange; +import org.apache.phoenix.schema.PColumn; +import org.apache.phoenix.schema.PColumnImpl; +import org.apache.phoenix.schema.PNameFactory; +import org.apache.phoenix.schema.RowKeyValueAccessor; +import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.TableRef; +import org.apache.phoenix.schema.tuple.ResultTuple; +import org.apache.phoenix.schema.tuple.Tuple; +import org.apache.phoenix.schema.types.PLong; +import org.apache.phoenix.trace.util.Tracing; +import org.apache.phoenix.util.ByteUtil; +import org.apache.phoenix.util.SizedUtil; +import org.cloudera.htrace.Sampler; +import org.cloudera.htrace.TraceScope; + +public class TraceQueryPlan implements QueryPlan { + + private TraceStatement traceStatement = null; + private PhoenixStatement stmt = null; + private StatementContext context = null; + private boolean first = true; + + private static final RowProjector TRACE_PROJECTOR; + static { + List<ExpressionProjector> projectedColumns = new ArrayList<ExpressionProjector>(); + PColumn column = + new PColumnImpl(PNameFactory.newName(MetricInfo.TRACE.columnName), null, + PLong.INSTANCE, null, null, false, 0, SortOrder.getDefault(), 0, null, + false, null); + List<PColumn> columns = new ArrayList<PColumn>(); + columns.add(column); + Expression expression = + new RowKeyColumnExpression(column, new RowKeyValueAccessor(columns, 0)); + projectedColumns.add(new ExpressionProjector(MetricInfo.TRACE.columnName, "", expression, + true)); + int estimatedByteSize = SizedUtil.KEY_VALUE_SIZE + PLong.INSTANCE.getByteSize(); + TRACE_PROJECTOR = new RowProjector(projectedColumns, estimatedByteSize, false); + } + + public TraceQueryPlan(TraceStatement traceStatement, PhoenixStatement stmt) { + this.traceStatement = traceStatement; + this.stmt = stmt; + this.context = new StatementContext(stmt); + } + + @Override + public StatementContext getContext() { + return this.context; + } + + @Override + public ParameterMetaData getParameterMetaData() { + return PhoenixParameterMetaData.EMPTY_PARAMETER_META_DATA; + } + + @Override + public ExplainPlan getExplainPlan() throws SQLException { + return ExplainPlan.EMPTY_PLAN; + } + + @Override + public ResultIterator iterator() throws SQLException { + final PhoenixConnection conn = stmt.getConnection(); + if (conn.getTraceScope() == null && !traceStatement.isTraceOn()) { + return ResultIterator.EMPTY_ITERATOR; + } + return new ResultIterator() { + + @Override + public void close() throws SQLException { + } + + @Override + public Tuple next() throws SQLException { + if(!first) return null; + TraceScope traceScope = conn.getTraceScope(); + if(traceStatement.isTraceOn()) { + if(!conn.getSampler().equals(Sampler.ALWAYS)) { + conn.setSampler(Sampler.ALWAYS); + } + if (traceScope == null) { + traceScope = Tracing.startNewSpan(conn, "Enabling trace"); + conn.setTraceScope(traceScope); + } + } else { + if (traceScope != null) { + conn.getTraceScope().close(); + conn.setTraceScope(null); + } + conn.setSampler(Sampler.NEVER); + } + if(traceScope == null) return null; + first = false; + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + ParseNodeFactory factory = new ParseNodeFactory(); + LiteralParseNode literal = + factory.literal(traceScope.getSpan().getTraceId()); + LiteralExpression expression = + LiteralExpression.newConstant(literal.getValue(), PLong.INSTANCE, + Determinism.ALWAYS); + expression.evaluate(null, ptr); + byte[] rowKey = ByteUtil.copyKeyBytesIfNecessary(ptr); + Cell cell = + CellUtil.createCell(rowKey, HConstants.EMPTY_BYTE_ARRAY, + HConstants.EMPTY_BYTE_ARRAY, System.currentTimeMillis(), + Type.Put.getCode(), HConstants.EMPTY_BYTE_ARRAY); + List<Cell> cells = new ArrayList<Cell>(1); + cells.add(cell); + return new ResultTuple(Result.create(cells)); + } + + @Override + public void explain(List<String> planSteps) { + } + }; + } + + @Override + public long getEstimatedSize() { + return PLong.INSTANCE.getByteSize(); + } + + @Override + public TableRef getTableRef() { + return null; + } + + @Override + public RowProjector getProjector() { + return TRACE_PROJECTOR; + } + + @Override + public Integer getLimit() { + return null; + } + + @Override + public OrderBy getOrderBy() { + return OrderBy.EMPTY_ORDER_BY; + } + + @Override + public GroupBy getGroupBy() { + return GroupBy.EMPTY_GROUP_BY; + } + + @Override + public List<KeyRange> getSplits() { + return Collections.emptyList(); + } + + @Override + public List<List<Scan>> getScans() { + return Collections.emptyList(); + } + + @Override + public FilterableStatement getStatement() { + return null; + } + + @Override + public boolean isDegenerate() { + return false; + } + + @Override + public boolean isRowKeyOrdered() { + return false; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/0440aca5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java index de9e323..630c8f5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixConnection.java @@ -93,6 +93,7 @@ import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SQLCloseable; import org.apache.phoenix.util.SQLCloseables; import org.cloudera.htrace.Sampler; +import org.cloudera.htrace.TraceScope; import com.google.common.base.Objects; import com.google.common.base.Strings; @@ -130,6 +131,7 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd private final String datePattern; private final String timePattern; private final String timestampPattern; + private TraceScope traceScope = null; private boolean isClosed = false; private Sampler<?> sampler; @@ -257,6 +259,10 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd return this.sampler; } + public void setSampler(Sampler<?> sampler) throws SQLException { + this.sampler = sampler; + } + public Map<String, String> getCustomTracingAnnotations() { return customTracingAnnotations; } @@ -408,6 +414,9 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd } try { try { + if (traceScope != null) { + traceScope.close(); + } closeStatements(); } finally { services.removeConnection(this); @@ -776,4 +785,12 @@ public class PhoenixConnection implements Connection, org.apache.phoenix.jdbc.Jd public KeyValueBuilder getKeyValueBuilder() { return this.services.getKeyValueBuilder(); } + + public TraceScope getTraceScope() { + return traceScope; + } + + public void setTraceScope(TraceScope traceScope) { + this.traceScope = traceScope; + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/0440aca5/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java index 4ca5bb5..4a23ab7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java @@ -55,6 +55,7 @@ import org.apache.phoenix.compile.StatementNormalizer; import org.apache.phoenix.compile.StatementPlan; import org.apache.phoenix.compile.SubqueryRewriter; import org.apache.phoenix.compile.SubselectRewriter; +import org.apache.phoenix.compile.TraceQueryPlan; import org.apache.phoenix.compile.UpsertCompiler; import org.apache.phoenix.coprocessor.MetaDataProtocol; import org.apache.phoenix.exception.BatchUpdateExecution; @@ -93,6 +94,7 @@ import org.apache.phoenix.parse.SQLParser; import org.apache.phoenix.parse.SelectStatement; import org.apache.phoenix.parse.TableName; import org.apache.phoenix.parse.TableNode; +import org.apache.phoenix.parse.TraceStatement; import org.apache.phoenix.parse.UpdateStatisticsStatement; import org.apache.phoenix.parse.UpsertStatement; import org.apache.phoenix.query.KeyRange; @@ -124,6 +126,8 @@ import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.SQLCloseable; import org.apache.phoenix.util.SQLCloseables; import org.apache.phoenix.util.ServerUtil; +import org.cloudera.htrace.Sampler; +import org.cloudera.htrace.TraceScope; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -690,6 +694,19 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho } } + private static class ExecutableTraceStatement extends TraceStatement implements CompilableStatement { + + public ExecutableTraceStatement(boolean isTraceOn) { + super(isTraceOn); + } + + @SuppressWarnings("unchecked") + @Override + public QueryPlan compilePlan(final PhoenixStatement stmt, Sequence.ValueOp seqAction) throws SQLException { + return new TraceQueryPlan(this, stmt); + } + } + private static class ExecutableUpdateStatisticsStatement extends UpdateStatisticsStatement implements CompilableStatement { public ExecutableUpdateStatisticsStatement(NamedTableNode table, StatisticsCollectionScope scope) { @@ -882,7 +899,12 @@ public class PhoenixStatement implements Statement, SQLCloseable, org.apache.pho public AlterIndexStatement alterIndex(NamedTableNode indexTableNode, String dataTableName, boolean ifExists, PIndexState state) { return new ExecutableAlterIndexStatement(indexTableNode, dataTableName, ifExists, state); } - + + @Override + public TraceStatement trace(boolean isTraceOn) { + return new ExecutableTraceStatement(isTraceOn); + } + @Override public ExplainStatement explain(BindableStatement statement) { return new ExecutableExplainStatement(statement); http://git-wip-us.apache.org/repos/asf/phoenix/blob/0440aca5/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java index ddfaa03..4e8f792 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java @@ -325,6 +325,10 @@ public class ParseNodeFactory { return new AlterIndexStatement(indexTableNode, dataTableName, ifExists, state); } + public TraceStatement trace(boolean isTraceOn) { + return new TraceStatement(isTraceOn); + } + public TableName table(String schemaName, String tableName) { return TableName.createNormalized(schemaName,tableName); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/0440aca5/phoenix-core/src/main/java/org/apache/phoenix/parse/TraceStatement.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/TraceStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/TraceStatement.java new file mode 100644 index 0000000..7460a85 --- /dev/null +++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/TraceStatement.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.parse; + +import org.apache.phoenix.jdbc.PhoenixStatement.Operation; + +public class TraceStatement implements BindableStatement { + + private boolean traceOn = false; + + public TraceStatement(boolean isOn) { + this.traceOn = isOn; + } + + @Override + public int getBindCount() { + return 0; + } + + @Override + public Operation getOperation() { + return Operation.QUERY; + } + + public boolean isTraceOn() { + return traceOn == true; + } +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/0440aca5/phoenix-core/src/main/java/org/apache/phoenix/trace/util/Tracing.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/util/Tracing.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/util/Tracing.java index a46d4e8..7cd55e8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/trace/util/Tracing.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/util/Tracing.java @@ -263,4 +263,14 @@ public class Tracing { } initialized = true; } + + public static boolean isTraceOn(String traceOption) { + Preconditions.checkArgument(traceOption != null); + if(traceOption.equalsIgnoreCase("ON")) return true; + if(traceOption.equalsIgnoreCase("OFF")) return false; + else { + throw new IllegalArgumentException("Unknown tracing option: " + traceOption); + } + } + }
