This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 9451dd0 Add Active Compactions page to Monitor (#2283)
9451dd0 is described below
commit 9451dd0dda6b4d9ebb5203e8314782233b28b75f
Author: Mike Miller <mmil...@apache.org>
AuthorDate: Thu Sep 23 10:38:43 2021 -0400
Add Active Compactions page to Monitor (#2283)
---
.../java/org/apache/accumulo/monitor/Monitor.java | 62 +++++++++++++++++++
.../monitor/rest/compactions/CompactionInfo.java | 53 +++++++++++++++++
.../monitor/rest/compactions/Compactions.java | 43 ++++++++++++++
.../CompactionsResource.java} | 28 ++++-----
.../accumulo/monitor/rest/scans/ScansResource.java | 2 +-
.../org/apache/accumulo/monitor/view/WebViews.java | 20 ++++++-
.../accumulo/monitor/resources/js/compactions.js | 69 ++++++++++++++++++++++
.../accumulo/monitor/templates/compactions.ftl | 39 ++++++++++++
.../apache/accumulo/monitor/templates/navbar.ftl | 1 +
9 files changed, 300 insertions(+), 17 deletions(-)
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
index 378923f..6217347 100644
--- a/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
+++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/Monitor.java
@@ -53,6 +53,7 @@ import
org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo;
import org.apache.accumulo.core.master.thrift.TableInfo;
import org.apache.accumulo.core.master.thrift.TabletServerStatus;
import org.apache.accumulo.core.rpc.ThriftUtil;
+import org.apache.accumulo.core.tabletserver.thrift.ActiveCompaction;
import org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
import org.apache.accumulo.core.tabletserver.thrift.TabletClientService.Client;
import org.apache.accumulo.core.trace.TraceUtil;
@@ -484,6 +485,17 @@ public class Monitor extends AbstractServer implements
HighlyAvailableService {
}
}).start();
+ Threads.createThread("Compaction fetcher", () -> {
+ while (true) {
+ try {
+ fetchCompactions();
+ } catch (Exception e) {
+ log.warn("{}", e.getMessage(), e);
+ }
+ sleepUninterruptibly(5, TimeUnit.SECONDS);
+ }
+ }).start();
+
monitorInitialized.set(true);
}
@@ -553,7 +565,24 @@ public class Monitor extends AbstractServer implements
HighlyAvailableService {
}
}
+ public static class CompactionStats {
+ public final long count;
+ public final Long oldest;
+ public final long fetched;
+
+ CompactionStats(List<ActiveCompaction> active) {
+ this.count = active.size();
+ long oldest = -1;
+ for (ActiveCompaction a : active) {
+ oldest = Math.max(oldest, a.age);
+ }
+ this.oldest = oldest < 0 ? null : oldest;
+ this.fetched = System.currentTimeMillis();
+ }
+ }
+
private final Map<HostAndPort,ScanStats> allScans = new HashMap<>();
+ private final Map<HostAndPort,CompactionStats> allCompactions = new
HashMap<>();
private final RecentLogs recentLogs = new RecentLogs();
public Map<HostAndPort,ScanStats> getScans() {
@@ -562,6 +591,12 @@ public class Monitor extends AbstractServer implements
HighlyAvailableService {
}
}
+ public Map<HostAndPort,CompactionStats> getCompactions() {
+ synchronized (allCompactions) {
+ return new HashMap<>(allCompactions);
+ }
+ }
+
private void fetchScans() throws Exception {
ServerContext context = getContext();
for (String server : context.instanceOperations().getTabletServers()) {
@@ -589,6 +624,33 @@ public class Monitor extends AbstractServer implements
HighlyAvailableService {
}
}
+ private void fetchCompactions() throws Exception {
+ ServerContext context = getContext();
+ for (String server : context.instanceOperations().getTabletServers()) {
+ final HostAndPort parsedServer = HostAndPort.fromString(server);
+ Client tserver = ThriftUtil.getTServerClient(parsedServer, context);
+ try {
+ var compacts = tserver.getActiveCompactions(null, context.rpcCreds());
+ synchronized (allCompactions) {
+ allCompactions.put(parsedServer, new CompactionStats(compacts));
+ }
+ } catch (Exception ex) {
+ log.debug("Failed to get active compactions from {}", server, ex);
+ } finally {
+ ThriftUtil.returnClient(tserver);
+ }
+ }
+ // Age off old compaction information
+ var entryIter = allCompactions.entrySet().iterator();
+ long now = System.currentTimeMillis();
+ while (entryIter.hasNext()) {
+ var entry = entryIter.next();
+ if (now - entry.getValue().fetched > 5 * 60 * 1000) {
+ entryIter.remove();
+ }
+ }
+ }
+
/**
* Get the monitor lock in ZooKeeper
*/
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/compactions/CompactionInfo.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/compactions/CompactionInfo.java
new file mode 100644
index 0000000..5bb6bff
--- /dev/null
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/compactions/CompactionInfo.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.monitor.rest.compactions;
+
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+
+/**
+ * Generates a compaction info JSON object
+ *
+ * @since 2.1.0
+ */
+public class CompactionInfo {
+
+ // Variable names become JSON keys
+ public String server;
+
+ public long count;
+ public Long oldest;
+
+ public CompactionInfo() {}
+
+ /**
+ * Stores new compaction information
+ *
+ * @param tserverInfo
+ * status of the tserver
+ * @param count
+ * number of compactions
+ * @param oldest
+ * time of oldest compaction
+ */
+ public CompactionInfo(TabletServerStatus tserverInfo, long count, Long
oldest) {
+ this.server = tserverInfo.getName();
+ this.count = count;
+ this.oldest = oldest;
+ }
+}
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/compactions/Compactions.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/compactions/Compactions.java
new file mode 100644
index 0000000..57c5817
--- /dev/null
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/compactions/Compactions.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.monitor.rest.compactions;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Generates a new list of compactions as a JSON object
+ *
+ * @since 2.1.0
+ */
+public class Compactions {
+
+ // Variable names become JSON keys
+ public List<CompactionInfo> compactions = new ArrayList<>();
+
+ /**
+ * Adds a new compactionInfo to the list
+ *
+ * @param compactionInfo
+ * Compaction info to add
+ */
+ public void addCompaction(CompactionInfo compactionInfo) {
+ compactions.add(compactionInfo);
+ }
+}
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/scans/ScansResource.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/compactions/CompactionsResource.java
similarity index 71%
copy from
server/monitor/src/main/java/org/apache/accumulo/monitor/rest/scans/ScansResource.java
copy to
server/monitor/src/main/java/org/apache/accumulo/monitor/rest/compactions/CompactionsResource.java
index 28c0188..9af29b0 100644
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/scans/ScansResource.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/compactions/CompactionsResource.java
@@ -16,7 +16,7 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.accumulo.monitor.rest.scans;
+package org.apache.accumulo.monitor.rest.compactions;
import java.util.Map;
@@ -30,42 +30,40 @@ import
org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo;
import org.apache.accumulo.core.master.thrift.TabletServerStatus;
import org.apache.accumulo.core.util.HostAndPort;
import org.apache.accumulo.monitor.Monitor;
-import org.apache.accumulo.monitor.Monitor.ScanStats;
/**
- * Generate a new Scan list JSON object
+ * Generate a new Compaction list JSON object
*
- * @since 2.0.0
+ * @since 2.1.0
*/
-@Path("/scans")
+@Path("/compactions")
@Produces({MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML})
-public class ScansResource {
+public class CompactionsResource {
@Inject
private Monitor monitor;
/**
- * Generates a new JSON object with scan information
+ * Generates a new JSON object with compaction information
*
* @return Scan JSON object
*/
@GET
- public Scans getTables() {
- Scans scans = new Scans();
+ public Compactions getActiveCompactions() {
+ Compactions compactions = new Compactions();
ManagerMonitorInfo mmi = monitor.getMmi();
if (mmi == null) {
- return scans;
+ return compactions;
}
- Map<HostAndPort,ScanStats> entry = monitor.getScans();
+ Map<HostAndPort,Monitor.CompactionStats> entry = monitor.getCompactions();
- // Adds new scans to the array
for (TabletServerStatus tserverInfo : mmi.getTServerInfo()) {
- ScanStats stats = entry.get(HostAndPort.fromString(tserverInfo.name));
+ var stats = entry.get(HostAndPort.fromString(tserverInfo.name));
if (stats != null) {
- scans.addScan(new ScanInformation(tserverInfo, stats.scanCount,
stats.oldestScan));
+ compactions.addCompaction(new CompactionInfo(tserverInfo, stats.count,
stats.oldest));
}
}
- return scans;
+ return compactions;
}
}
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/scans/ScansResource.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/scans/ScansResource.java
index 28c0188..c43c44b 100644
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/scans/ScansResource.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/rest/scans/ScansResource.java
@@ -50,7 +50,7 @@ public class ScansResource {
* @return Scan JSON object
*/
@GET
- public Scans getTables() {
+ public Scans getActiveScans() {
Scans scans = new Scans();
ManagerMonitorInfo mmi = monitor.getMmi();
if (mmi == null) {
diff --git
a/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java
b/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java
index ddaa631..dbbb6dc 100644
---
a/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java
+++
b/server/monitor/src/main/java/org/apache/accumulo/monitor/view/WebViews.java
@@ -184,7 +184,7 @@ public class WebViews {
public Map<String,Object> getScans() {
Map<String,Object> model = getModel();
- model.put("title", "Scans");
+ model.put("title", "Active scans");
model.put("template", "scans.ftl");
model.put("js", "scans.js");
@@ -192,6 +192,24 @@ public class WebViews {
}
/**
+ * Returns the compactions template
+ *
+ * @return Scans model
+ */
+ @GET
+ @Path("compactions")
+ @Template(name = "/default.ftl")
+ public Map<String,Object> getCompactions() {
+
+ Map<String,Object> model = getModel();
+ model.put("title", "Active Compactions");
+ model.put("template", "compactions.ftl");
+ model.put("js", "compactions.js");
+
+ return model;
+ }
+
+ /**
* Returns the bulk import template
*
* @return Bulk Import model
diff --git
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/compactions.js
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/compactions.js
new file mode 100644
index 0000000..570dbb6
--- /dev/null
+++
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/resources/js/compactions.js
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ var compactionsList;
+ /**
+ * Creates active compactions table
+ */
+ $(document).ready(function() {
+ // Create a table for compactions list
+ compactionsList = $('#compactionsList').DataTable({
+ "ajax": {
+ "url": '/rest/compactions',
+ "dataSrc": "compactions"
+ },
+ "stateSave": true,
+ "dom": 't<"align-left"l>p',
+ "columnDefs": [
+ { "targets": "duration",
+ "render": function ( data, type, row ) {
+ if(type === 'display') data = timeDuration(data);
+ return data;
+ }
+ }
+ ],
+ "columns": [
+ { "data": "server",
+ "type": "html",
+ "render": function ( data, type, row, meta ) {
+ if(type === 'display') {
+ data = '<a href="/tservers?s=' + row.server + '">' +
row.server + '</a>';
+ }
+ return data;
+ }
+ },
+ { "data": "count" },
+ { "data": "oldest" }
+ ]
+ });
+ });
+
+ /**
+ * Generates the compactions table
+ */
+ function refreshCompactionsTable() {
+ if(compactionsList) compactionsList.ajax.reload(null, false ); // user
paging is not reset on reload
+ }
+
+ /**
+ * Used to redraw the page
+ */
+ function refresh() {
+ refreshCompactionsTable();
+ }
diff --git
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/compactions.ftl
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/compactions.ftl
new file mode 100644
index 0000000..22ea801
--- /dev/null
+++
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/compactions.ftl
@@ -0,0 +1,39 @@
+<#--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+ <div class="row">
+ <div class="col-xs-12">
+ <h3>${title}</h3>
+ </div>
+ </div>
+ <div class="row">
+ <div class="col-xs-12">
+ <table id="compactionsList" class="table table-bordered
table-striped table-condensed">
+ <thead>
+ <tr>
+ <th class="firstcell">Server </th>
+ <th title="Number of compactions presently
running"># </th>
+ <th class="duration" title="The age of the oldest compaction
on this server.">Oldest Age </th>
+ </tr>
+ </thead>
+ <tbody></tbody>
+ </table>
+ </div>
+ </div>
diff --git
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/navbar.ftl
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/navbar.ftl
index e6f3da0..f180b6d 100644
---
a/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/navbar.ftl
+++
b/server/monitor/src/main/resources/org/apache/accumulo/monitor/templates/navbar.ftl
@@ -51,6 +51,7 @@
Activity <span class="caret"></span>
</a>
<ul class="dropdown-menu">
+ <li><a href="/compactions">Active Compactions</a></li>
<li><a href="/scans">Active Scans</a></li>
<li><a href="/bulkImports">Bulk Imports</a></li>
<li><a href="/replication">Replication</a></li>
