Repository: tajo Updated Branches: refs/heads/master 6d852081e -> 3919896e1
TAJO-1693: Rearrange metric names. Closes #642 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/3919896e Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/3919896e Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/3919896e Branch: refs/heads/master Commit: 3919896e10822929ead6ceb797acfc067137ef05 Parents: 6d85208 Author: Hyunsik Choi <[email protected]> Authored: Thu Jul 23 16:11:32 2015 +0900 Committer: Hyunsik Choi <[email protected]> Committed: Thu Jul 23 16:11:32 2015 +0900 ---------------------------------------------------------------------- CHANGES | 2 + pom.xml | 1 + tajo-core/pom.xml | 10 +- .../org/apache/tajo/master/GlobalEngine.java | 20 +- .../tajo/master/QueryCoordinatorService.java | 7 +- .../java/org/apache/tajo/master/TajoMaster.java | 33 ++- .../apache/tajo/master/exec/QueryExecutor.java | 5 +- .../tajo/metrics/CatalogMetricsGaugeSet.java | 56 ---- .../tajo/metrics/ClusterResourceMetricSet.java | 104 +++++++ .../metrics/WorkerResourceMetricsGaugeSet.java | 74 ----- .../querymaster/QueryMasterManagerService.java | 1 - .../tajo/querymaster/QueryMasterTask.java | 21 -- .../org/apache/tajo/session/SessionManager.java | 5 + .../apache/tajo/util/metrics/TajoMetrics.java | 68 ++--- .../tajo/util/metrics/TajoSystemMetrics.java | 21 +- .../java/org/apache/tajo/worker/TajoWorker.java | 11 +- .../tajo/worker/TajoWorkerManagerService.java | 1 - .../org/apache/tajo/worker/TaskContainer.java | 2 - .../org/apache/tajo/worker/TaskExecutor.java | 2 - .../tajo/util/metrics/TestSystemMetrics.java | 37 ++- .../apache/tajo/worker/MockWorkerContext.java | 4 +- .../tajo/worker/TestNodeResourceManager.java | 2 +- .../tajo/worker/TestNodeStatusUpdater.java | 2 +- .../apache/tajo/worker/TestTaskExecutor.java | 2 +- .../org/apache/tajo/worker/TestTaskManager.java | 2 +- tajo-dist/pom.xml | 1 + tajo-metrics/pom.xml | 271 +++++++++++++++++++ .../org/apache/tajo/metrics/MetricsUtil.java | 96 +++++++ .../src/main/proto/master_metrics.proto | 57 ++++ tajo-metrics/src/main/proto/node_metrics.proto | 36 +++ .../apache/tajo/metrics/TestMetricsUtil.java | 53 ++++ tajo-project/pom.xml | 6 + 32 files changed, 742 insertions(+), 271 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 21799d2..f81db07 100644 --- a/CHANGES +++ b/CHANGES @@ -399,6 +399,8 @@ Release 0.11.0 - unreleased SUB TASKS + TAJO-1693: Rearrange metric names. (hyunsik) + TAJO-1496: Remove legacy CSVFile. (jinho) TAJO-1691: Refactor visitors and planner to throw TajoException. (hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index b6f0d16..e4c8331 100644 --- a/pom.xml +++ b/pom.xml @@ -93,6 +93,7 @@ <module>tajo-dist</module> <module>tajo-thirdparty/asm</module> <module>tajo-cli</module> + <module>tajo-metrics</module> </modules> <build> http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-core/pom.xml b/tajo-core/pom.xml index bd52e12..7489503 100644 --- a/tajo-core/pom.xml +++ b/tajo-core/pom.xml @@ -33,7 +33,7 @@ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> <antlr4.visitor>true</antlr4.visitor> <antlr4.listener>true</antlr4.listener> - <metrics.version>3.0.1</metrics.version> + <metrics.version>3.1.0</metrics.version> <maven.fork.count>3</maven.fork.count> </properties> @@ -286,6 +286,10 @@ <groupId>org.apache.tajo</groupId> <artifactId>tajo-ws-rs</artifactId> </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-metrics</artifactId> + </dependency> <dependency> <groupId>org.apache.hadoop</groupId> @@ -410,12 +414,12 @@ </exclusions> </dependency> <dependency> - <groupId>com.codahale.metrics</groupId> + <groupId>io.dropwizard.metrics</groupId> <artifactId>metrics-core</artifactId> <version>${metrics.version}</version> </dependency> <dependency> - <groupId>com.codahale.metrics</groupId> + <groupId>io.dropwizard.metrics</groupId> <artifactId>metrics-jvm</artifactId> <version>${metrics.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java index 00d346a..8bfe65a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java @@ -34,21 +34,28 @@ import org.apache.tajo.algebra.JsonHelper; import org.apache.tajo.catalog.CatalogService; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.exception.ReturnStateUtil; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.parser.SQLAnalyzer; import org.apache.tajo.engine.parser.SQLSyntaxError; import org.apache.tajo.engine.query.QueryContext; import org.apache.tajo.exception.ExceptionUtil; +import org.apache.tajo.exception.ReturnStateUtil; import org.apache.tajo.master.TajoMaster.MasterContext; import org.apache.tajo.master.exec.DDLExecutor; import org.apache.tajo.master.exec.QueryExecutor; -import org.apache.tajo.plan.*; +import org.apache.tajo.metrics.Master; +import org.apache.tajo.plan.IllegalQueryStatusException; +import org.apache.tajo.plan.LogicalOptimizer; +import org.apache.tajo.plan.LogicalPlan; +import org.apache.tajo.plan.LogicalPlanner; import org.apache.tajo.plan.logical.InsertNode; import org.apache.tajo.plan.logical.LogicalRootNode; import org.apache.tajo.plan.logical.NodeType; import org.apache.tajo.plan.util.PlannerUtil; -import org.apache.tajo.plan.verifier.*; +import org.apache.tajo.plan.verifier.LogicalPlanVerifier; +import org.apache.tajo.plan.verifier.PreLogicalPlanVerifier; +import org.apache.tajo.plan.verifier.SyntaxErrorUtil; +import org.apache.tajo.plan.verifier.VerificationState; import org.apache.tajo.session.Session; import org.apache.tajo.storage.TablespaceManager; import org.apache.tajo.util.CommonTestingUtil; @@ -57,7 +64,6 @@ import java.io.IOException; import java.sql.SQLException; import java.util.concurrent.TimeUnit; -import static org.apache.tajo.exception.ReturnStateUtil.returnError; import static org.apache.tajo.ipc.ClientProtos.SubmitQueryResponse; public class GlobalEngine extends AbstractService { @@ -175,6 +181,8 @@ public class GlobalEngine extends AbstractService { Expr planningContext; try { + context.getMetrics().counter(Master.Query.SUBMITTED).inc(); + if (isJson) { planningContext = buildExpressionFromJson(query); } else { @@ -190,7 +198,8 @@ public class GlobalEngine extends AbstractService { } catch (Throwable t) { ExceptionUtil.printStackTraceIfError(LOG, t); - context.getSystemMetrics().counter("Query", "errorQuery").inc(); + context.getMetrics().counter(Master.Query.ERROR).inc(); + SubmitQueryResponse.Builder responseBuilder = SubmitQueryResponse.newBuilder(); responseBuilder.setUserName(queryContext.get(SessionVars.USERNAME)); responseBuilder.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); @@ -206,7 +215,6 @@ public class GlobalEngine extends AbstractService { public Expr buildExpressionFromSql(String sql, Session session) throws InterruptedException, IOException, IllegalQueryStatusException { - context.getSystemMetrics().counter("Query", "totalQuery").inc(); try { if (session.getQueryCache() == null) { return analyzer.parse(sql); http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/main/java/org/apache/tajo/master/QueryCoordinatorService.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/QueryCoordinatorService.java b/tajo-core/src/main/java/org/apache/tajo/master/QueryCoordinatorService.java index 3b04fc5..fc7e0e3 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/QueryCoordinatorService.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/QueryCoordinatorService.java @@ -34,11 +34,13 @@ import org.apache.tajo.rpc.AsyncRpcServer; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos; import org.apache.tajo.rpc.protocolrecords.PrimitiveProtos.BoolProto; import org.apache.tajo.util.NetUtils; +import org.apache.tajo.util.ProtoUtil; import java.net.InetSocketAddress; import java.util.Collection; import static org.apache.tajo.ResourceProtos.*; +import static org.apache.tajo.util.ProtoUtil.TRUE; public class QueryCoordinatorService extends AbstractService { private final static Log LOG = LogFactory.getLog(QueryCoordinatorService.class); @@ -49,9 +51,6 @@ public class QueryCoordinatorService extends AbstractService { private AsyncRpcServer server; private InetSocketAddress bindAddress; - private final BoolProto BOOL_TRUE = BoolProto.newBuilder().setValue(true).build(); - private final BoolProto BOOL_FALSE = BoolProto.newBuilder().setValue(false).build(); - public QueryCoordinatorService(TajoMaster.MasterContext context) { super(QueryCoordinatorService.class.getName()); this.context = context; @@ -106,7 +105,7 @@ public class QueryCoordinatorService extends AbstractService { command = queryManager.queryHeartbeat(request); TajoHeartbeatResponse.Builder builder = TajoHeartbeatResponse.newBuilder(); - builder.setHeartbeatResult(BOOL_TRUE); + builder.setHeartbeatResult(TRUE); if(command != null) { builder.setResponseCommand(command); } http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java index 9327c59..0a4ac2c 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java @@ -18,10 +18,14 @@ package org.apache.tajo.master; +import com.codahale.metrics.Gauge; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.*; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.service.CompositeService; @@ -39,8 +43,8 @@ import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.engine.function.FunctionLoader; import org.apache.tajo.function.FunctionSignature; import org.apache.tajo.master.rm.TajoResourceManager; -import org.apache.tajo.metrics.CatalogMetricsGaugeSet; -import org.apache.tajo.metrics.WorkerResourceMetricsGaugeSet; +import org.apache.tajo.metrics.ClusterResourceMetricSet; +import org.apache.tajo.metrics.Master; import org.apache.tajo.rpc.RpcChannelFactory; import org.apache.tajo.rpc.RpcClientManager; import org.apache.tajo.rpc.RpcConstants; @@ -73,13 +77,10 @@ import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; import static org.apache.tajo.TajoConstants.DEFAULT_TABLESPACE_NAME; public class TajoMaster extends CompositeService { - private static final String METRICS_GROUP_NAME = "tajomaster"; /** Class Logger */ private static final Log LOG = LogFactory.getLog(TajoMaster.class); - public static final int SHUTDOWN_HOOK_PRIORITY = 30; - /** rw-r--r-- */ @SuppressWarnings("OctalInteger") final public static FsPermission TAJO_ROOT_DIR_PERMISSION = FsPermission.createImmutable((short) 0755); @@ -134,6 +135,8 @@ public class TajoMaster extends CompositeService { private HistoryReader historyReader; + private static final long CLUSTER_STARTUP_TIME = System.currentTimeMillis(); + public TajoMaster() throws Exception { super(TajoMaster.class.getName()); } @@ -214,11 +217,17 @@ public class TajoMaster extends CompositeService { } private void initSystemMetrics() { - systemMetrics = new TajoSystemMetrics(systemConf, METRICS_GROUP_NAME, getMasterName()); + systemMetrics = new TajoSystemMetrics(systemConf, Master.class, getMasterName()); systemMetrics.start(); - systemMetrics.register("resource", new WorkerResourceMetricsGaugeSet(context)); - systemMetrics.register("catalog", new CatalogMetricsGaugeSet(context)); + systemMetrics.register(Master.Cluster.UPTIME, new Gauge<Long>() { + @Override + public Long getValue() { + return context.getClusterUptime(); + } + }); + + systemMetrics.register(Master.Cluster.class, new ClusterResourceMetricSet(context)); } private void initResourceManager() throws Exception { @@ -423,6 +432,10 @@ public class TajoMaster extends CompositeService { return clock; } + public long getClusterUptime() { + return getClock().getTime() - CLUSTER_STARTUP_TIME; + } + public QueryManager getQueryJobManager() { return queryManager; } @@ -451,7 +464,7 @@ public class TajoMaster extends CompositeService { return tajoMasterService; } - public TajoSystemMetrics getSystemMetrics() { + public TajoSystemMetrics getMetrics() { return systemMetrics; } http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java index b6e5806..ceb3c4a 100644 --- a/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/master/exec/QueryExecutor.java @@ -61,6 +61,7 @@ import org.apache.tajo.plan.verifier.VerifyException; import org.apache.tajo.session.Session; import org.apache.tajo.storage.*; import org.apache.tajo.util.ProtoUtil; +import org.apache.tajo.util.metrics.TajoMetrics; import org.apache.tajo.worker.TaskAttemptContext; import java.io.IOException; @@ -103,7 +104,6 @@ public class QueryExecutor { } else if (PlannerUtil.checkIfDDLPlan(rootNode)) { - context.getSystemMetrics().counter("Query", "numDDLQuery").inc(); ddlExecutor.execute(queryContext, plan); response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); response.setState(OK); @@ -158,7 +158,6 @@ public class QueryExecutor { } } - context.getSystemMetrics().counter("Query", "numDDLQuery").inc(); response.setQueryId(QueryIdFactory.NULL_QUERY_ID.getProto()); response.setState(OK); } @@ -488,7 +487,7 @@ public class QueryExecutor { space.prepareTable(rootNode.getChild()); } - context.getSystemMetrics().counter("Query", "numDMLQuery").inc(); + hookManager.doHooks(queryContext, plan); QueryManager queryManager = this.context.getQueryJobManager(); http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/main/java/org/apache/tajo/metrics/CatalogMetricsGaugeSet.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/metrics/CatalogMetricsGaugeSet.java b/tajo-core/src/main/java/org/apache/tajo/metrics/CatalogMetricsGaugeSet.java deleted file mode 100644 index 82ebe29..0000000 --- a/tajo-core/src/main/java/org/apache/tajo/metrics/CatalogMetricsGaugeSet.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.metrics; - -import com.codahale.metrics.Gauge; -import com.codahale.metrics.Metric; -import com.codahale.metrics.MetricSet; -import org.apache.tajo.master.TajoMaster; - -import java.util.HashMap; -import java.util.Map; - -import static org.apache.tajo.TajoConstants.DEFAULT_DATABASE_NAME; - -public class CatalogMetricsGaugeSet implements MetricSet { - TajoMaster.MasterContext tajoMasterContext; - public CatalogMetricsGaugeSet(TajoMaster.MasterContext tajoMasterContext) { - this.tajoMasterContext = tajoMasterContext; - } - - @Override - public Map<String, Metric> getMetrics() { - Map<String, Metric> metricsMap = new HashMap<String, Metric>(); - metricsMap.put("numTables", new Gauge<Integer>() { - @Override - public Integer getValue() { - return tajoMasterContext.getCatalog().getAllTableNames(DEFAULT_DATABASE_NAME).size(); - } - }); - - metricsMap.put("numFunctions", new Gauge<Integer>() { - @Override - public Integer getValue() { - return tajoMasterContext.getCatalog().getFunctions().size(); - } - }); - - return metricsMap; - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/main/java/org/apache/tajo/metrics/ClusterResourceMetricSet.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/metrics/ClusterResourceMetricSet.java b/tajo-core/src/main/java/org/apache/tajo/metrics/ClusterResourceMetricSet.java new file mode 100644 index 0000000..7454d21 --- /dev/null +++ b/tajo-core/src/main/java/org/apache/tajo/metrics/ClusterResourceMetricSet.java @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.metrics; + +import com.codahale.metrics.Gauge; +import com.codahale.metrics.Metric; +import com.codahale.metrics.MetricSet; +import org.apache.tajo.master.TajoMaster; +import org.apache.tajo.master.rm.NodeStatus; +import org.apache.tajo.master.rm.NodeState; +import org.apache.tajo.metrics.Master.Cluster; + +import java.util.HashMap; +import java.util.Map; + +public class ClusterResourceMetricSet implements MetricSet { + TajoMaster.MasterContext masterContext; + public ClusterResourceMetricSet(TajoMaster.MasterContext masterContext) { + this.masterContext = masterContext; + } + + @Override + public Map<String, Metric> getMetrics() { + Map<String, Metric> metricsMap = new HashMap<String, Metric>(); + + metricsMap.put(Cluster.TOTAL_NODES.name(), new Gauge<Integer>() { + @Override + public Integer getValue() { + return masterContext.getResourceManager().getNodes().size(); + } + }); + + metricsMap.put(Cluster.ACTIVE_NODES.name(), new Gauge<Integer>() { + @Override + public Integer getValue() { + return getNumWorkers(NodeState.RUNNING); + } + }); + + metricsMap.put(Cluster.LOST_NODES.name(), new Gauge<Integer>() { + @Override + public Integer getValue() { + return getNumWorkers(NodeState.LOST); + } + }); + + metricsMap.put(Cluster.TOTAL_MEMORY.name(), new Gauge<Integer>() { + @Override + public Integer getValue() { + return masterContext.getResourceManager().getScheduler().getMaximumResourceCapability().getMemory(); + } + }); + + metricsMap.put(Cluster.FREE_MEMORY.name(), new Gauge<Integer>() { + @Override + public Integer getValue() { + return masterContext.getResourceManager().getScheduler().getClusterResource().getMemory(); + } + }); + + metricsMap.put(Cluster.TOTAL_VCPU.name(), new Gauge<Integer>() { + @Override + public Integer getValue() { + return masterContext.getResourceManager().getScheduler().getMaximumResourceCapability().getVirtualCores(); + } + }); + + metricsMap.put(Cluster.FREE_VCPU.name(), new Gauge<Integer>() { + @Override + public Integer getValue() { + return masterContext.getResourceManager().getScheduler().getClusterResource().getVirtualCores(); + } + }); + + return metricsMap; + } + + protected int getNumWorkers(NodeState status) { + int numWorkers = 0; + for(NodeStatus eachNodeStatus : masterContext.getResourceManager().getNodes().values()) { + if(eachNodeStatus.getState() == status) { + numWorkers++; + } + } + + return numWorkers; + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/main/java/org/apache/tajo/metrics/WorkerResourceMetricsGaugeSet.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/metrics/WorkerResourceMetricsGaugeSet.java b/tajo-core/src/main/java/org/apache/tajo/metrics/WorkerResourceMetricsGaugeSet.java deleted file mode 100644 index 200d689..0000000 --- a/tajo-core/src/main/java/org/apache/tajo/metrics/WorkerResourceMetricsGaugeSet.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.metrics; - -import com.codahale.metrics.Gauge; -import com.codahale.metrics.Metric; -import com.codahale.metrics.MetricSet; -import org.apache.tajo.master.TajoMaster; -import org.apache.tajo.master.rm.NodeStatus; -import org.apache.tajo.master.rm.NodeState; - -import java.util.HashMap; -import java.util.Map; - -public class WorkerResourceMetricsGaugeSet implements MetricSet { - TajoMaster.MasterContext tajoMasterContext; - public WorkerResourceMetricsGaugeSet(TajoMaster.MasterContext tajoMasterContext) { - this.tajoMasterContext = tajoMasterContext; - } - - @Override - public Map<String, Metric> getMetrics() { - Map<String, Metric> metricsMap = new HashMap<String, Metric>(); - metricsMap.put("totalWorkers", new Gauge<Integer>() { - @Override - public Integer getValue() { - return tajoMasterContext.getResourceManager().getNodes().size(); - } - }); - - metricsMap.put("liveWorkers", new Gauge<Integer>() { - @Override - public Integer getValue() { - return getNumWorkers(NodeState.RUNNING); - } - }); - - metricsMap.put("deadWorkers", new Gauge<Integer>() { - @Override - public Integer getValue() { - return getNumWorkers(NodeState.LOST); - } - }); - - return metricsMap; - } - - protected int getNumWorkers(NodeState status) { - int numWorkers = 0; - for(NodeStatus eachNodeStatus : tajoMasterContext.getResourceManager().getNodes().values()) { - if(eachNodeStatus.getState() == status) { - numWorkers++; - } - } - - return numWorkers; - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterManagerService.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterManagerService.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterManagerService.java index 62216aa..05e7165 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterManagerService.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterManagerService.java @@ -230,7 +230,6 @@ public class QueryMasterManagerService extends CompositeService public void executeQuery(RpcController controller, QueryExecutionRequest request, RpcCallback<PrimitiveProtos.NullProto> done) { - workerContext.getWorkerSystemMetrics().counter("querymaster", "numQuery").inc(); QueryId queryId = new QueryId(request.getQueryId()); LOG.info("Receive executeQuery request:" + queryId); http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java index 61e2dea..611560d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java +++ b/tajo-core/src/main/java/org/apache/tajo/querymaster/QueryMasterTask.java @@ -101,8 +101,6 @@ public class QueryMasterTask extends CompositeService { private volatile boolean isStopped; - private TajoMetrics queryMetrics; - private Throwable initError; private NodeResource allocation; @@ -149,12 +147,6 @@ public class QueryMasterTask extends CompositeService { dispatcher.register(TaskSchedulerEvent.EventType.class, new TaskSchedulerDispatcher()); dispatcher.register(LocalTaskEventType.class, new LocalTaskEventHandler()); - try { - queryMetrics = new TajoMetrics(queryId.toString()); - } catch (Throwable t) { - LOG.error(t.getMessage(), t); - initError = t; - } super.serviceInit(systemConf); } @@ -191,15 +183,6 @@ public class QueryMasterTask extends CompositeService { cleanupQuery(getQueryId()); } - if (queryMetrics != null) { - queryMasterContext.getEventExecutor().submit(new Runnable() { - @Override - public void run() { - queryMetrics.report(new MetricsConsoleReporter()); - } - }); - } - super.serviceStop(); LOG.info("Stopped QueryMasterTask:" + queryId); } @@ -554,10 +537,6 @@ public class QueryMasterTask extends CompositeService { return query.getProgress(); } - public TajoMetrics getQueryMetrics() { - return queryMetrics; - } - /** * A key is worker id, and a value is a worker connection information. */ http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/main/java/org/apache/tajo/session/SessionManager.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/session/SessionManager.java b/tajo-core/src/main/java/org/apache/tajo/session/SessionManager.java index 9c49fe1..a69533f 100644 --- a/tajo-core/src/main/java/org/apache/tajo/session/SessionManager.java +++ b/tajo-core/src/main/java/org/apache/tajo/session/SessionManager.java @@ -56,6 +56,11 @@ public class SessionManager extends CompositeService implements EventHandler<Ses super.serviceStop(); } + public int currentNum() { + return this.sessions.size(); + } + + private void assertSessionExistence(String sessionId) throws InvalidSessionException { if (!sessions.containsKey(sessionId)) { throw new InvalidSessionException(sessionId); http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoMetrics.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoMetrics.java b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoMetrics.java index 0e378b2..46ff401 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoMetrics.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoMetrics.java @@ -19,12 +19,13 @@ package org.apache.tajo.util.metrics; import com.codahale.metrics.*; +import com.google.common.base.Preconditions; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.tajo.metrics.MetricsUtil; import org.apache.tajo.util.metrics.reporter.TajoMetricsReporter; import java.util.Map; -import java.util.SortedMap; import java.util.concurrent.atomic.AtomicBoolean; public class TajoMetrics { @@ -67,67 +68,30 @@ public class TajoMetrics { return metricRegistry.getMetrics(); } - public SortedMap<String, Gauge> getGuageMetrics(MetricFilter filter) { - if(filter == null) { - filter = MetricFilter.ALL; - } - return metricRegistry.getGauges(filter); + public void register(Enum<?> item, Gauge gauge) { + Preconditions.checkArgument(metricsGroupName.equals(MetricsUtil.getGroupName(item))); + metricRegistry.register(MetricsUtil.getCanonicalName(item), gauge); } - public SortedMap<String, Counter> getCounterMetrics(MetricFilter filter) { - if(filter == null) { - filter = MetricFilter.ALL; - } - return metricRegistry.getCounters(filter); - } + public void register(Class<? extends Enum<?>> context, MetricSet metricSet) { + Preconditions.checkArgument(metricsGroupName.equals(MetricsUtil.getGroupName(context))); - public SortedMap<String, Histogram> getHistogramMetrics(MetricFilter filter) { - if(filter == null) { - filter = MetricFilter.ALL; - } - return metricRegistry.getHistograms(filter); + metricRegistry.register(MetricsUtil.getCanonicalContextName(context), metricSet); } - public SortedMap<String, Meter> getMeterMetrics(MetricFilter filter) { - if(filter == null) { - filter = MetricFilter.ALL; - } - return metricRegistry.getMeters(filter); + public Counter counter(Enum<?> item) { + return metricRegistry.counter(MetricsUtil.getCanonicalName(item)); } - public SortedMap<String, Timer> getTimerMetrics(MetricFilter filter) { - if(filter == null) { - filter = MetricFilter.ALL; - } - return metricRegistry.getTimers(filter); - } - - public void register(String contextName, MetricSet metricSet) { - metricRegistry.register(MetricRegistry.name(metricsGroupName, contextName), metricSet); + public Histogram histogram(Enum<?> item) { + return metricRegistry.histogram(MetricsUtil.getCanonicalName(item)); } - public void register(String contextName, String itemName, Gauge gauge) { - metricRegistry.register(makeMetricsName(metricsGroupName, contextName, itemName), gauge); + public Meter meter(Enum<?> item) { + return metricRegistry.meter(MetricsUtil.getCanonicalName(item)); } - public Counter counter(String contextName, String itemName) { - return metricRegistry.counter(makeMetricsName(metricsGroupName, contextName, itemName)); + public Timer timer(Enum<?> item) { + return metricRegistry.timer(MetricsUtil.getCanonicalName(item)); } - - public Histogram histogram(String contextName, String itemName) { - return metricRegistry.histogram(makeMetricsName(metricsGroupName, contextName, itemName)); - } - - public Meter meter(String contextName, String itemName) { - return metricRegistry.meter(makeMetricsName(metricsGroupName, contextName, itemName)); - } - - public Timer timer(String contextName, String itemName) { - return metricRegistry.timer(makeMetricsName(metricsGroupName, contextName, itemName)); - } - - public static String makeMetricsName(String metricsGroupName, String contextName, String itemName) { - return MetricRegistry.name(metricsGroupName, contextName, itemName); - } - } http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java index 4192ca0..ecdf5ef 100644 --- a/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java +++ b/tajo-core/src/main/java/org/apache/tajo/util/metrics/TajoSystemMetrics.java @@ -31,6 +31,7 @@ import org.apache.commons.configuration.reloading.FileChangedReloadingStrategy; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.metrics.MetricsUtil; import org.apache.tajo.util.metrics.reporter.TajoMetricsScheduledReporter; import java.util.*; @@ -50,8 +51,8 @@ public class TajoSystemMetrics extends TajoMetrics { private String metricsPropertyFileName; - public TajoSystemMetrics(TajoConf tajoConf, String metricsGroupName, String hostAndPort) { - super(metricsGroupName); + public TajoSystemMetrics(TajoConf tajoConf, Class clazz, String hostAndPort) { + super(MetricsUtil.getGroupName(clazz)); this.hostAndPort = hostAndPort; try { @@ -65,8 +66,8 @@ public class TajoSystemMetrics extends TajoMetrics { LOG.warn(e.getMessage(), e); } - //PropertiesConfiguration fire configurationChanged after getXXX() - //So neeaded calling getXXX periodically + // PropertiesConfiguration fire configurationChanged after getXXX() + // So neeaded calling getXXX periodically propertyChangeChecker = new Thread() { public void run() { while(!stop.get()) { @@ -110,21 +111,21 @@ public class TajoSystemMetrics extends TajoMetrics { public void start() { setMetricsReporter(metricsGroupName); - String jvmMetricsName = metricsGroupName + "-jvm"; + final String jvmMetricsName = metricsGroupName + "-JVM"; setMetricsReporter(jvmMetricsName); if(!inited) { - metricRegistry.register(MetricRegistry.name(jvmMetricsName, "Heap"), new MemoryUsageGaugeSet()); - metricRegistry.register(MetricRegistry.name(jvmMetricsName, "File"), new FileDescriptorRatioGauge()); + metricRegistry.register(MetricRegistry.name(jvmMetricsName, "MEMORY"), new MemoryUsageGaugeSet()); + metricRegistry.register(MetricRegistry.name(jvmMetricsName, "FILE"), new FileDescriptorRatioGauge()); metricRegistry.register(MetricRegistry.name(jvmMetricsName, "GC"), new GarbageCollectorMetricSet()); - metricRegistry.register(MetricRegistry.name(jvmMetricsName, "Thread"), new ThreadStatesGaugeSet()); - metricRegistry.register(MetricRegistry.name(jvmMetricsName, "Log"), new LogEventGaugeSet()); + metricRegistry.register(MetricRegistry.name(jvmMetricsName, "THREAD"), new ThreadStatesGaugeSet()); + metricRegistry.register(MetricRegistry.name(jvmMetricsName, "LOG"), new LogEventGaugeSet()); } inited = true; } private void setMetricsReporter(String groupName) { - //reporter name -> class name + // reporter name -> class name Map<String, String> reporters = new HashMap<String, String>(); List<String> reporterNames = metricsProps.getList(groupName + ".reporters"); http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java index a95698f..1119b59 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorker.java @@ -37,6 +37,7 @@ import org.apache.tajo.catalog.FunctionDesc; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.engine.function.FunctionLoader; import org.apache.tajo.function.FunctionSignature; +import org.apache.tajo.metrics.Node; import org.apache.tajo.rpc.RpcClientManager; import org.apache.tajo.rpc.RpcConstants; import org.apache.tajo.service.ServiceTracker; @@ -240,10 +241,10 @@ public class TajoWorker extends CompositeService { } private void initWorkerMetrics() { - workerSystemMetrics = new TajoSystemMetrics(systemConf, "worker", workerContext.getWorkerName()); + workerSystemMetrics = new TajoSystemMetrics(systemConf, Node.class, workerContext.getWorkerName()); workerSystemMetrics.start(); - workerSystemMetrics.register("querymaster", "runningQueries", new Gauge<Integer>() { + workerSystemMetrics.register(Node.QueryMaster.RUNNING_QM, new Gauge<Integer>() { @Override public Integer getValue() { if(queryMasterManagerService != null) { @@ -254,7 +255,7 @@ public class TajoWorker extends CompositeService { } }); - workerSystemMetrics.register("task", "runningTasks", new Gauge<Integer>() { + workerSystemMetrics.register(Node.Tasks.RUNNING_TASKS, new Gauge<Integer>() { @Override public Integer getValue() { if(taskExecutor != null) { @@ -397,7 +398,7 @@ public class TajoWorker extends CompositeService { LocalDirAllocator getLocalDirAllocator(); - TajoSystemMetrics getWorkerSystemMetrics(); + TajoSystemMetrics getMetrics(); HashShuffleAppenderManager getHashShuffleAppenderManager(); @@ -505,7 +506,7 @@ public class TajoWorker extends CompositeService { } } - public TajoSystemMetrics getWorkerSystemMetrics() { + public TajoSystemMetrics getMetrics() { return workerSystemMetrics; } http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java index 7752211..4595e5d 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TajoWorkerManagerService.java @@ -109,7 +109,6 @@ public class TajoWorkerManagerService extends CompositeService public void allocateTasks(RpcController controller, BatchAllocationRequest request, RpcCallback<BatchAllocationResponse> done) { - workerContext.getWorkerSystemMetrics().counter("query", "allocationRequestNum").inc(); workerContext.getNodeResourceManager().getDispatcher(). getEventHandler().handle(new NodeResourceAllocateEvent(request, done)); } http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/main/java/org/apache/tajo/worker/TaskContainer.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskContainer.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskContainer.java index f717c07..bd28bb7 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskContainer.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskContainer.java @@ -47,8 +47,6 @@ public class TaskContainer implements Runnable { try { task = executor.getNextTask(); - task.getExecutionBlockContext().getWorkerContext().getWorkerSystemMetrics().counter("query", "task").inc(); - if (LOG.isDebugEnabled()) { LOG.debug(sequenceId + TaskContainer.class.getSimpleName() + " got task:" + task.getTaskContext().getTaskId()); http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java b/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java index eef4a2d..1d6e2b8 100644 --- a/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java +++ b/tajo-core/src/main/java/org/apache/tajo/worker/TaskExecutor.java @@ -175,8 +175,6 @@ public class TaskExecutor extends AbstractService implements EventHandler<TaskSt } taskQueue.put(task); runningTasks.incrementAndGet(); - context.getWorkerContext().getWorkerSystemMetrics() - .histogram("tasks", "running").update(runningTasks.get()); } else { LOG.warn("Release duplicate task resource: " + event.getAllocatedResource()); stopTask(event.getTaskAttemptId()); http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/test/java/org/apache/tajo/util/metrics/TestSystemMetrics.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/util/metrics/TestSystemMetrics.java b/tajo-core/src/test/java/org/apache/tajo/util/metrics/TestSystemMetrics.java index 0ce2646..8751df9 100644 --- a/tajo-core/src/test/java/org/apache/tajo/util/metrics/TestSystemMetrics.java +++ b/tajo-core/src/test/java/org/apache/tajo/util/metrics/TestSystemMetrics.java @@ -23,6 +23,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.metrics.Master; +import org.apache.tajo.metrics.MetricsUtil; import org.apache.tajo.util.CommonTestingUtil; import org.apache.tajo.util.metrics.reporter.TajoMetricsScheduledReporter; import org.junit.After; @@ -54,13 +56,13 @@ public class TestSystemMetrics { out.write("reporter.file=org.apache.tajo.util.metrics.reporter.MetricsFileScheduledReporter\n".getBytes()); out.write("reporter.console=org.apache.tajo.util.metrics.reporter.MetricsConsoleScheduledReporter\n".getBytes()); - out.write("test-file-group-jvm.reporters=console\n".getBytes()); - out.write("test-file-group.reporters=file\n".getBytes()); + out.write("MASTER-JVM.reporters=console\n".getBytes()); + out.write("MASTER.reporters=file\n".getBytes()); out.write("test-console-group.reporters=console\n".getBytes()); out.write("test-find-console-group.reporters=console,file\n".getBytes()); - out.write(("test-file-group.file.filename=" + metricsOutputFile.toUri().getPath() + "\n").getBytes()); - out.write("test-file-group.file.period=5\n".getBytes()); + out.write(("MASTER.file.filename=" + metricsOutputFile.toUri().getPath() + "\n").getBytes()); + out.write("MASTER.file.period=5\n".getBytes()); out.close(); } @@ -68,7 +70,8 @@ public class TestSystemMetrics { public void testMetricsReporter() throws Exception { TajoConf tajoConf = new TajoConf(); tajoConf.set("tajo.metrics.property.file", testPropertyFile.toUri().getPath()); - TajoSystemMetrics tajoSystemMetrics = new TajoSystemMetrics(tajoConf, "test-file-group", "localhost"); + TajoSystemMetrics tajoSystemMetrics = new TajoSystemMetrics(tajoConf, org.apache.tajo.metrics.Master.class, + "localhost"); tajoSystemMetrics.start(); Collection<TajoMetricsScheduledReporter> reporters = tajoSystemMetrics.getMetricsReporters(); @@ -79,36 +82,40 @@ public class TestSystemMetrics { assertEquals(5, reporter.getPeriod()); for(int i = 0; i < 10; i++) { - tajoSystemMetrics.counter("test-group01", "test-item1").inc(); - tajoSystemMetrics.counter("test-group01", "test-item2").inc(2); - tajoSystemMetrics.counter("test-group02", "test-item1").inc(3); + tajoSystemMetrics.counter(Master.Query.FAILED).inc(); + tajoSystemMetrics.counter(Master.Query.COMPLETED).inc(2); + tajoSystemMetrics.counter(Master.Cluster.ACTIVE_NODES).inc(3); } SortedMap<String, Counter> counterMap = tajoSystemMetrics.getRegistry().getCounters(); - Counter counter1 = counterMap.get("test-file-group.test-group01.test-item1"); + Counter counter1 = counterMap.get("MASTER.QUERY.FAILED"); assertNotNull(counter1); assertEquals(10, counter1.getCount()); - Counter counter2 = counterMap.get("test-file-group.test-group01.test-item2"); + Counter counter2 = counterMap.get("MASTER.QUERY.COMPLETED"); assertNotNull(counter2); assertEquals(20, counter2.getCount()); + Counter counter3 = counterMap.get("MASTER.CLUSTER.ACTIVE_NODES"); + assertNotNull(counter3); + assertEquals(30, counter3.getCount()); + //test findMetricsItemGroup method Map<String, Map<String, Counter>> groupItems = reporter.findMetricsItemGroup(counterMap); assertEquals(2, groupItems.size()); - Map<String, Counter> group01Items = groupItems.get("test-file-group.test-group01"); + Map<String, Counter> group01Items = groupItems.get(MetricsUtil.getCanonicalContextName(Master.Query.class)); assertEquals(2, group01Items.size()); - counter1 = group01Items.get("test-item1"); + counter1 = group01Items.get(Master.Query.FAILED.name()); assertNotNull(counter1); assertEquals(10, counter1.getCount()); - counter2 = group01Items.get("test-item2"); + counter2 = group01Items.get(Master.Query.COMPLETED.name()); assertNotNull(counter2); assertEquals(20, counter2.getCount()); - Map<String, Counter> group02Items = groupItems.get("test-file-group.test-group02"); + Map<String, Counter> group02Items = groupItems.get(MetricsUtil.getCanonicalContextName(Master.Cluster.class)); assertEquals(1, group02Items.size()); reporter.report(); @@ -116,7 +123,7 @@ public class TestSystemMetrics { BufferedReader reader = new BufferedReader(new InputStreamReader( new FileInputStream(metricsOutputFile.toUri().getPath()))); - String line = null; + String line; List<String> lines = new ArrayList<String>(); while((line = reader.readLine()) != null) { http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/test/java/org/apache/tajo/worker/MockWorkerContext.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/MockWorkerContext.java b/tajo-core/src/test/java/org/apache/tajo/worker/MockWorkerContext.java index f57fa85..25f3dca 100644 --- a/tajo-core/src/test/java/org/apache/tajo/worker/MockWorkerContext.java +++ b/tajo-core/src/test/java/org/apache/tajo/worker/MockWorkerContext.java @@ -72,10 +72,10 @@ public abstract class MockWorkerContext implements TajoWorker.WorkerContext { } @Override - public TajoSystemMetrics getWorkerSystemMetrics() { + public TajoSystemMetrics getMetrics() { if (tajoSystemMetrics == null) { - tajoSystemMetrics = new TajoSystemMetrics(getConf(), "test-file-group", "localhost"); + tajoSystemMetrics = new TajoSystemMetrics(getConf(), org.apache.tajo.metrics.Node.class, "localhost"); tajoSystemMetrics.start(); } return tajoSystemMetrics; http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeResourceManager.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeResourceManager.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeResourceManager.java index 7539beb..1193478 100644 --- a/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeResourceManager.java +++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeResourceManager.java @@ -122,7 +122,7 @@ public class TestNodeResourceManager { @Override protected void serviceStop() throws Exception { - workerContext.getWorkerSystemMetrics().stop(); + workerContext.getMetrics().stop(); super.serviceStop(); } }; http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeStatusUpdater.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeStatusUpdater.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeStatusUpdater.java index 707aa49..ac4b7dd 100644 --- a/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeStatusUpdater.java +++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestNodeStatusUpdater.java @@ -110,7 +110,7 @@ public class TestNodeStatusUpdater { @Override protected void serviceStop() throws Exception { - workerContext.getWorkerSystemMetrics().stop(); + workerContext.getMetrics().stop(); super.serviceStop(); } }; http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java index c0f5bb2..7cb7e71 100644 --- a/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java +++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskExecutor.java @@ -122,7 +122,7 @@ public class TestTaskExecutor { @Override protected void serviceStop() throws Exception { - workerContext.getWorkerSystemMetrics().stop(); + workerContext.getMetrics().stop(); super.serviceStop(); } }; http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskManager.java ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskManager.java b/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskManager.java index 630f3f7..1d84afe 100644 --- a/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskManager.java +++ b/tajo-core/src/test/java/org/apache/tajo/worker/TestTaskManager.java @@ -127,7 +127,7 @@ public class TestTaskManager { @Override protected void serviceStop() throws Exception { - workerContext.getWorkerSystemMetrics().stop(); + workerContext.getMetrics().stop(); super.serviceStop(); } }; http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-dist/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-dist/pom.xml b/tajo-dist/pom.xml index cd08b3d..3b8f947 100644 --- a/tajo-dist/pom.xml +++ b/tajo-dist/pom.xml @@ -142,6 +142,7 @@ run cp -r $ROOT/tajo-catalog/target/tajo-catalog-${project.version}/* . run cp -r $ROOT/tajo-storage/target/tajo-storage-${project.version}/* . run cp -r $ROOT/tajo-pullserver/target/tajo-pullserver-${project.version}.jar . + run cp -r $ROOT/tajo-metrics/target/tajo-metrics-${project.version}.jar . run cp -r $ROOT/tajo-core/target/tajo-core-${project.version}.jar . run cp -r $ROOT/tajo-core/target/lib . run cp -r ${project.basedir}/src/main/bin . http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-metrics/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-metrics/pom.xml b/tajo-metrics/pom.xml new file mode 100644 index 0000000..123138e --- /dev/null +++ b/tajo-metrics/pom.xml @@ -0,0 +1,271 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>tajo-project</artifactId> + <groupId>org.apache.tajo</groupId> + <version>0.11.0-SNAPSHOT</version> + <relativePath>../tajo-project</relativePath> + </parent> + <artifactId>tajo-metrics</artifactId> + <packaging>jar</packaging> + <name>Tajo Metrics</name> + <description>This module contains an utility for Tajo system metrics collection.</description> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> + </properties> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <source>1.6</source> + <target>1.6</target> + <encoding>${project.build.sourceEncoding}</encoding> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + <executions> + <execution> + <phase>verify</phase> + <goals> + <goal>check</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>create-protobuf-generated-sources-directory</id> + <phase>initialize</phase> + <configuration> + <target> + <mkdir dir="target/generated-sources/proto" /> + </target> + </configuration> + <goals> + <goal>run</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <version>1.2</version> + <executions> + <execution> + <id>generate-sources</id> + <phase>generate-sources</phase> + <configuration> + <executable>protoc</executable> + <arguments> + <argument>-Isrc/main/proto/</argument> + <argument>src/main/proto/master_metrics.proto</argument> + <argument>src/main/proto/node_metrics.proto</argument> + <argument>--java_out=target/generated-sources/proto</argument> + </arguments> + </configuration> + <goals> + <goal>exec</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <version>1.5</version> + <executions> + <execution> + <id>add-source</id> + <phase>generate-sources</phase> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <source>target/generated-sources/proto</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <version>2.2</version> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <executions> + <execution> + <id>copy-dependencies</id> + <phase>package</phase> + <goals> + <goal>copy-dependencies</goal> + </goals> + <configuration> + <includeScope>runtime</includeScope> + <outputDirectory>${project.build.directory}/lib</outputDirectory> + <overWriteReleases>false</overWriteReleases> + <overWriteSnapshots>false</overWriteSnapshots> + <overWriteIfNewer>true</overWriteIfNewer> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + </plugin> + </plugins> + </build> + + + <dependencies> + <dependency> + <groupId>com.google.protobuf</groupId> + <artifactId>protobuf-java</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + + <profiles> + <profile> + <id>docs</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <executions> + <execution> + <!-- build javadoc jars per jar for publishing to maven --> + <id>module-javadocs</id> + <phase>package</phase> + <goals> + <goal>jar</goal> + </goals> + <configuration> + <destDir>${project.build.directory}</destDir> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> + <id>dist</id> + <activation> + <activeByDefault>false</activeByDefault> + <property> + <name>tar|rpm|deb</name> + </property> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <executions> + <execution> + <id>dist</id> + <phase>package</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <echo file="${project.build.directory}/dist-layout-stitching.sh"> + run() { + echo "\$ ${@}" + "${@}" + res=$? + if [ $res != 0 ]; then + echo + echo "Failed!" + echo + exit $res + fi + } + + ROOT=`cd ${basedir}/..;pwd` + echo + echo "Current directory `pwd`" + echo + run rm -rf ${project.artifactId}-${project.version} + run mkdir ${project.artifactId}-${project.version} + run cd ${project.artifactId}-${project.version} + run cp -r ${basedir}/target/${project.artifactId}-${project.version}*.jar . + echo + echo "Tajo Metrics dist layout available at: ${project.build.directory}/${project.artifactId}-${project.version}" + echo + </echo> + <exec executable="sh" dir="${project.build.directory}" failonerror="true"> + <arg line="./dist-layout-stitching.sh" /> + </exec> + </target> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> + + <reporting> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-report-plugin</artifactId> + <version>2.15</version> + </plugin> + </plugins> + </reporting> +</project> + http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-metrics/src/main/java/org/apache/tajo/metrics/MetricsUtil.java ---------------------------------------------------------------------- diff --git a/tajo-metrics/src/main/java/org/apache/tajo/metrics/MetricsUtil.java b/tajo-metrics/src/main/java/org/apache/tajo/metrics/MetricsUtil.java new file mode 100644 index 0000000..082244c --- /dev/null +++ b/tajo-metrics/src/main/java/org/apache/tajo/metrics/MetricsUtil.java @@ -0,0 +1,96 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.metrics; + +import java.util.regex.Pattern; + +/** + * Each metric name can be divided into group name, context name, and item name. + */ +public class MetricsUtil { + public static final String DELIMITER = "."; + + static final Pattern pattern; + + static { + pattern = Pattern.compile("\\."); + } + + private static void checkMetricsClass(Class clazz) { + if (!clazz.getName().substring(0, 23).equals("org.apache.tajo.metrics")) { + throw new RuntimeException("Invalid Metrics Class: " + clazz.getName()); + } + } + + public static String getGroupName(Class clazz) { + checkMetricsClass(clazz); + + + String [] names = pattern.split(clazz.getCanonicalName().substring(24)); + if (names.length > 0) { + return names[0].toUpperCase(); + } else { + if (clazz.isEnum()) { + return clazz.getSimpleName().toUpperCase(); + } else { + throw new RuntimeException("Invalid Metrics Class: " + clazz.getName()); + } + } + } + + public static String getGroupName(Enum<?> e) { + checkMetricsClass(e.getClass()); + + return pattern.split(e.getClass().getCanonicalName().substring(24))[0].toUpperCase(); + } + + public static String getCanonicalContextName(Class<? extends Enum<?>> clazz) { + checkMetricsClass(clazz); + + if (!clazz.isEnum()) { + throw new RuntimeException("Invalid Context Enum: " + clazz.getName()); + } + + return getGroupName(clazz) + DELIMITER + clazz.getSimpleName().toUpperCase(); + } + + public static String getContextName(Class<? extends Enum<?>> clazz) { + checkMetricsClass(clazz); + + if (!clazz.isEnum()) { + throw new RuntimeException("Invalid Context Enum: " + clazz.getName()); + } + + return clazz.getSimpleName().toUpperCase(); + } + + public static String getCanonicalName(Enum<?> e) { + checkMetricsClass(e.getClass()); + + String [] groupAndContextNames = pattern.split(e.getClass().getCanonicalName().substring(24)); + if (groupAndContextNames.length != 2) { + throw new RuntimeException("Invalid Metric Enum Type: " + e.getClass().getName()); + } + return (groupAndContextNames[0] + DELIMITER + groupAndContextNames[1] + DELIMITER + e.name()).toUpperCase(); + } + + public static String getSimpleName(Enum<?> e) { + return e.name(); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-metrics/src/main/proto/master_metrics.proto ---------------------------------------------------------------------- diff --git a/tajo-metrics/src/main/proto/master_metrics.proto b/tajo-metrics/src/main/proto/master_metrics.proto new file mode 100644 index 0000000..e3dbd12 --- /dev/null +++ b/tajo-metrics/src/main/proto/master_metrics.proto @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tajo.metrics; + +option java_package = "org.apache.tajo.metrics"; +option java_outer_classname = "Master"; +option java_generic_services = false; +option java_generate_equals_and_hash = true; + +// Category +// * 'Cluster' - About overall cluster and resources +// * 'Query' - Query submission, scheduler, and statistic information about queries + +enum Cluster { + UPTIME = 0; + + TOTAL_NODES = 1; + ACTIVE_NODES = 2; + LOST_NODES = 3; + + TOTAL_MEMORY = 4; + FREE_MEMORY = 5; + TOTAL_VCPU = 6; + FREE_VCPU = 7; +} + +// Query Coordinator +enum Query { + SUBMITTED = 1; + COMPLETED = 2; + RUNNING = 3; + ERROR = 4; + FAILED = 5; + KILLED = 6; + + NUM_PER_HOUR = 6; + MAX_IO_THROUGHPUT = 7; + AVG_IO_THROUGHPUT = 8; + + PENDING_QUERIES = 9; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-metrics/src/main/proto/node_metrics.proto ---------------------------------------------------------------------- diff --git a/tajo-metrics/src/main/proto/node_metrics.proto b/tajo-metrics/src/main/proto/node_metrics.proto new file mode 100644 index 0000000..74f6613 --- /dev/null +++ b/tajo-metrics/src/main/proto/node_metrics.proto @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package tajo.metrics.node; + +option java_package = "org.apache.tajo.metrics"; +option java_outer_classname = "Node"; +option java_generic_services = false; +option java_generate_equals_and_hash = true; + +// Category +// 'TASKS' - contains statistic information related to TaskManager, and each task execution +// 'QueryMaster' - contains statistic information related to QueryMaster and its manager. + +enum Tasks { + RUNNING_TASKS = 0; +} + +enum QueryMaster { + RUNNING_QM = 0; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-metrics/src/test/java/org/apache/tajo/metrics/TestMetricsUtil.java ---------------------------------------------------------------------- diff --git a/tajo-metrics/src/test/java/org/apache/tajo/metrics/TestMetricsUtil.java b/tajo-metrics/src/test/java/org/apache/tajo/metrics/TestMetricsUtil.java new file mode 100644 index 0000000..1d59e87 --- /dev/null +++ b/tajo-metrics/src/test/java/org/apache/tajo/metrics/TestMetricsUtil.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.metrics; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; + +public class TestMetricsUtil { + + @Test + public void testGroupName() { + assertEquals("MASTER", MetricsUtil.getGroupName(Master.class)); + assertEquals("MASTER", MetricsUtil.getGroupName(Master.Cluster.class)); + assertEquals("MASTER", MetricsUtil.getGroupName(Master.Cluster.ACTIVE_NODES)); + } + + @Test + public void testGetContextName() { + assertEquals("CLUSTER", MetricsUtil.getContextName(Master.Cluster.class)); + } + + @Test + public void testGetCanonicalContextName() { + assertEquals("MASTER.CLUSTER", MetricsUtil.getCanonicalContextName(Master.Cluster.class)); + } + + @Test + public void testGetCanonicalName() { + assertEquals("MASTER.CLUSTER.ACTIVE_NODES", MetricsUtil.getCanonicalName(Master.Cluster.ACTIVE_NODES)); + } + + @Test + public void testGetSimpleName() { + assertEquals("ACTIVE_NODES", MetricsUtil.getSimpleName(Master.Cluster.ACTIVE_NODES)); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/3919896e/tajo-project/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index d7e2f91..a85b611 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -836,6 +836,12 @@ <version>${tajo.version}</version> <type>jar</type> </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-metrics</artifactId> + <version>${tajo.version}</version> + <type>jar</type> + </dependency> <dependency> <groupId>org.apache.hadoop</groupId>
