[accumulo-examples] branch main updated: Make runmr script use HADOOP_HOME (#72)
This is an automated email from the ASF dual-hosted git repository.
mmiller pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git
The following commit(s) were added to refs/heads/main by this push:
new 7aa5a4d Make runmr script use HADOOP_HOME (#72)
7aa5a4d is described below
commit 7aa5a4d4cf69a7b534a2833c6a50017a0ad5542a
Author: Mike Miller
AuthorDate: Wed Mar 31 17:20:00 2021 -0400
Make runmr script use HADOOP_HOME (#72)
---
bin/runmr | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bin/runmr b/bin/runmr
index 9de164e..ee65fac 100755
--- a/bin/runmr
+++ b/bin/runmr
@@ -41,4 +41,4 @@ fi
"$ex_home"/bin/build
export HADOOP_USE_CLIENT_CLASSLOADER=true
-yarn jar $EXAMPLES_JAR_PATH "org.apache.accumulo.examples.$1" ${*:2}
+"$HADOOP_HOME"/bin/yarn jar $EXAMPLES_JAR_PATH
"org.apache.accumulo.examples.$1" ${*:2}
[accumulo-proxy] branch dependabot/maven/com.google.guava-guava-29.0-jre created (now 743a3c2)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch dependabot/maven/com.google.guava-guava-29.0-jre in repository https://gitbox.apache.org/repos/asf/accumulo-proxy.git. at 743a3c2 Bump guava from 28.2-jre to 29.0-jre No new revisions were added by this update.
[accumulo-wikisearch] branch dependabot/maven/com.google.guava-guava-29.0-jre created (now 8fb9da5)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch dependabot/maven/com.google.guava-guava-29.0-jre in repository https://gitbox.apache.org/repos/asf/accumulo-wikisearch.git. at 8fb9da5 Bump guava from 27.0-jre to 29.0-jre No new revisions were added by this update.
[accumulo-testing] branch dependabot/maven/com.google.guava-guava-29.0-jre created (now b1cbbfe)
This is an automated email from the ASF dual-hosted git repository. github-bot pushed a change to branch dependabot/maven/com.google.guava-guava-29.0-jre in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git. at b1cbbfe Bump guava from 26.0-jre to 29.0-jre No new revisions were added by this update.
[accumulo] branch main updated: Add created column to Fate Print command (#1980)
This is an automated email from the ASF dual-hosted git repository.
jmanno pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new aab34a9 Add created column to Fate Print command (#1980)
aab34a9 is described below
commit aab34a980d7a9ccad63797913b99b331c5cbbaae
Author: Jeffrey Manno
AuthorDate: Wed Mar 31 13:36:10 2021 -0400
Add created column to Fate Print command (#1980)
* Add timestamp of created time based on transaction Id's of FaTE operations
* Create a helper function to format the returned long value of the
timestamp
* Formatted created time within the `fate print` shell command
---
.../apache/accumulo/core/logging/FateLogger.java | 5
.../java/org/apache/accumulo/fate/AdminUtil.java | 31 +++---
.../java/org/apache/accumulo/fate/AgeOffStore.java | 5
.../org/apache/accumulo/fate/ReadOnlyStore.java| 5
.../org/apache/accumulo/fate/ReadOnlyTStore.java | 8 ++
.../java/org/apache/accumulo/fate/ZooStore.java| 13 +
.../java/org/apache/accumulo/fate/SimpleStore.java | 5
7 files changed, 68 insertions(+), 4 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java
b/core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java
index c856275..6121bce 100644
--- a/core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java
+++ b/core/src/main/java/org/apache/accumulo/core/logging/FateLogger.java
@@ -86,6 +86,11 @@ public class FateLogger {
}
@Override
+ public long timeCreated(long tid) {
+return store.timeCreated(tid);
+ }
+
+ @Override
public long create() {
long tid = store.create();
if (storeLog.isTraceEnabled())
diff --git a/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
b/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
index 3016d01..d4e45a8 100644
--- a/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
+++ b/core/src/main/java/org/apache/accumulo/fate/AdminUtil.java
@@ -20,8 +20,11 @@ package org.apache.accumulo.fate;
import static java.nio.charset.StandardCharsets.UTF_8;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Date;
import java.util.EnumSet;
import java.util.Formatter;
import java.util.HashMap;
@@ -80,9 +83,10 @@ public class AdminUtil {
private final List hlocks;
private final List wlocks;
private final String top;
+private final long timeCreated;
private TransactionStatus(Long tid, TStatus status, String debug,
List hlocks,
-List wlocks, String top) {
+List wlocks, String top, Long timeCreated) {
this.txid = tid;
this.status = status;
@@ -90,6 +94,7 @@ public class AdminUtil {
this.hlocks = Collections.unmodifiableList(hlocks);
this.wlocks = Collections.unmodifiableList(wlocks);
this.top = top;
+ this.timeCreated = timeCreated;
}
@@ -132,6 +137,21 @@ public class AdminUtil {
public String getTop() {
return top;
}
+
+/**
+ * @return The timestamp of when the operation was created in ISO format
wiht UTC timezone.
+ */
+public String getTimeCreatedFormatted() {
+ return timeCreated > 0 ? new
Date(timeCreated).toInstant().atZone(ZoneOffset.UTC)
+ .format(DateTimeFormatter.ISO_DATE_TIME) : "ERROR";
+}
+
+/**
+ * @return The unformatted form of the timestamp.
+ */
+public long getTimeCreated() {
+ return timeCreated;
+}
}
public static class FateStatus {
@@ -373,13 +393,15 @@ public class AdminUtil {
TStatus status = zs.getStatus(tid);
+ long timeCreated = zs.timeCreated(tid);
+
zs.unreserve(tid, 0);
if ((filterTxid != null && !filterTxid.contains(tid))
|| (filterStatus != null && !filterStatus.contains(status)))
continue;
- statuses.add(new TransactionStatus(tid, status, debug, hlocks, wlocks,
top));
+ statuses.add(new TransactionStatus(tid, status, debug, hlocks, wlocks,
top, timeCreated));
}
return new FateStatus(statuses, heldLocks, waitingLocks);
@@ -398,9 +420,10 @@ public class AdminUtil {
FateStatus fateStatus = getStatus(zs, zk, lockPath, filterTxid,
filterStatus);
for (TransactionStatus txStatus : fateStatus.getTransactions()) {
- fmt.format("txid: %s status: %-18s op: %-15s locked: %-15s locking:
%-15s top: %s%n",
+ fmt.format(
+ "txid: %s status: %-18s op: %-15s locked: %-15s locking: %-15s
top: %-15s created: %s%n",
txStatus.getTxid(), txStatus.getStatus(), txStatus.getDebug(),
txStatus.getHeldLocks(),
- txStatus.getWaitingLocks(), txStatus.getTop());
+ txStatus.getWaitingLocks(),
[accumulo-website] branch asf-site updated (728fd2d -> 1d4b480)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/accumulo-website.git. from 728fd2d Automatic Site Publish by Buildbot add 1d4b480 Automatic Site Publish by Buildbot No new revisions were added by this update. Summary of changes: output/1.10/examples/batch.html | 1 + output/1.10/examples/bloom.html | 1 + output/1.10/examples/bulkIngest.html | 1 + output/1.10/examples/classpath.html | 1 + output/1.10/examples/client.html | 1 + output/1.10/examples/combiner.html| 1 + output/1.10/examples/constraints.html | 1 + output/1.10/examples/dirlist.html | 1 + output/1.10/examples/export.html | 1 + output/1.10/examples/filedata.html| 1 + output/1.10/examples/filter.html | 1 + output/1.10/examples/helloworld.html | 1 + output/1.10/examples/index.html | 1 + output/1.10/examples/isolation.html | 1 + output/1.10/examples/mapred.html | 1 + output/1.10/examples/maxmutation.html | 1 + output/1.10/examples/regex.html | 1 + output/1.10/examples/reservations.html| 1 + output/1.10/examples/rgbalancer.html | 1 + output/1.10/examples/rowhash.html | 1 + output/1.10/examples/sample.html | 1 + output/1.10/examples/shard.html | 1 + output/1.10/examples/tabletofile.html | 1 + output/1.10/examples/terasort.html| 1 + output/1.10/examples/visibility.html | 1 + output/1.3/user_manual/Accumulo_Design.html | 1 + output/1.3/user_manual/Accumulo_Shell.html| 1 + output/1.3/user_manual/Administration.html| 1 + output/1.3/user_manual/Analytics.html | 1 + output/1.3/user_manual/Contents.html | 1 + output/1.3/user_manual/High_Speed_Ingest.html | 1 + output/1.3/user_manual/Introduction.html | 1 + output/1.3/user_manual/Security.html | 1 + output/1.3/user_manual/Shell_Commands.html| 1 + output/1.3/user_manual/Table_Configuration.html | 1 + output/1.3/user_manual/Table_Design.html | 1 + output/1.3/user_manual/Writing_Accumulo_Clients.html | 1 + output/1.3/user_manual/accumulo_user_manual.html | 1 + output/1.3/user_manual/examples/aggregation.html | 1 + output/1.3/user_manual/examples/batch.html| 1 + output/1.3/user_manual/examples/bloom.html| 1 + output/1.3/user_manual/examples/bulkIngest.html | 1 + output/1.3/user_manual/examples/constraints.html | 1 + output/1.3/user_manual/examples/dirlist.html | 1 + output/1.3/user_manual/examples/filter.html | 1 + output/1.3/user_manual/examples/helloworld.html | 1 + output/1.3/user_manual/examples/index.html| 1 + output/1.3/user_manual/examples/mapred.html | 1 + output/1.3/user_manual/examples/shard.html| 1 + output/1.3/user_manual/index.html | 1 + output/1.4/examples/batch.html| 1 + output/1.4/examples/bloom.html| 1 + output/1.4/examples/bulkIngest.html | 1 + output/1.4/examples/combiner.html | 1 + output/1.4/examples/constraints.html | 1 + output/1.4/examples/dirlist.html | 1 + output/1.4/examples/filedata.html | 1 + output/1.4/examples/filter.html | 1 + output/1.4/examples/helloworld.html | 1 + output/1.4/examples/index.html| 1 +
Buildbot success in on jekyll_websites
The Buildbot has detected a passing build on builder jekyll_websites while building accumulo. Full details are available at: https://ci2.apache.org/#builders/7/builds/435 Buildbot URL: https://ci2.apache.org/ Worker for this Build: bb_slave10_ubuntu Build Reason: Triggered jekyll auto-build via .asf.yaml by kturner Blamelist: asfinfra, commits@accumulo.apache.org Build succeeded! Sincerely, -The Buildbot
[accumulo-website] branch main updated: Add prominent link to public API docs (#268)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-website.git The following commit(s) were added to refs/heads/main by this push: new 2d336e9 Add prominent link to public API docs (#268) 2d336e9 is described below commit 2d336e9e1347ec2d78d2ece1c852bacfd792fe94 Author: Keith Turner AuthorDate: Wed Mar 31 11:33:07 2021 -0400 Add prominent link to public API docs (#268) --- _includes/nav.html | 1 + 1 file changed, 1 insertion(+) diff --git a/_includes/nav.html b/_includes/nav.html index c79080c..7dfde54 100644 --- a/_includes/nav.html +++ b/_includes/nav.html @@ -27,6 +27,7 @@ User Manual (2.x) Javadocs (2.0) +Public API Quickstart (1.x) Accumulo Maven Plugin User Manual (1.10)
[accumulo] branch 1451-external-compactions-feature updated: Fixes #1989: Optimization in CompactionCoordinator startup to find tserver hosting tablet that is being compacted
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 1451-external-compactions-feature
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to
refs/heads/1451-external-compactions-feature by this push:
new 3b9b9fa Fixes #1989: Optimization in CompactionCoordinator startup to
find tserver hosting tablet that is being compacted
3b9b9fa is described below
commit 3b9b9fa3105632aec5332cafbf857b5fdba4153d
Author: Dave Marion
AuthorDate: Wed Mar 31 15:31:07 2021 +
Fixes #1989: Optimization in CompactionCoordinator startup to find tserver
hosting tablet that is being compacted
---
.../coordinator/CompactionCoordinator.java | 79 +-
.../org/apache/accumulo/compactor/Compactor.java | 4 +-
2 files changed, 64 insertions(+), 19 deletions(-)
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index 7e1a42a..895d131 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@ -47,6 +47,9 @@ import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata.LocationType;
import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.securityImpl.thrift.TCredentials;
import org.apache.accumulo.core.tabletserver.thrift.CompactionStats;
@@ -213,27 +216,69 @@ public class CompactionCoordinator extends AbstractServer
LOG.info("Found {} running external compactions", running.size());
running.forEach((hp, job) -> {
// Find the tserver that has this compaction id
- // CBUG: Is there a more efficient way of finding the tablet server?
boolean matchFound = false;
- for (TServerInstance tsi : tservers) {
-TabletClientService.Client client = null;
-try {
- client = getTabletServerConnection(tsi);
- boolean tserverMatch =
client.isRunningExternalCompaction(TraceUtil.traceInfo(),
- getContext().rpcCreds(), job.getExternalCompactionId(),
job.getExtent());
- if (tserverMatch) {
-
RUNNING.put(ExternalCompactionId.of(job.getExternalCompactionId()),
-new RunningCompaction(job,
ExternalCompactionUtil.getHostPortString(hp), tsi));
-matchFound = true;
+
+ // Attempt to find the TServer hosting the tablet based on the
metadata table
+ // CBUG use #1974 for more efficient metadata reads
+ KeyExtent extent = KeyExtent.fromThrift(job.getExtent());
+ TabletMetadata tabletMetadata =
getContext().getAmple().readTablets().forTablet(extent)
+ .fetch(ColumnType.LOCATION,
ColumnType.PREV_ROW).build().stream().findFirst()
+ .orElse(null);
+
+ if (tabletMetadata != null &&
tabletMetadata.getExtent().equals(extent)
+ && tabletMetadata.getLocation() != null
+ && tabletMetadata.getLocation().getType() ==
LocationType.CURRENT) {
+
+TServerInstance tsi = tservers.stream()
+.filter(
+t ->
t.getHostAndPort().equals(tabletMetadata.getLocation().getHostAndPort()))
+.findFirst().orElse(null);
+
+if (null != tsi) {
+ TabletClientService.Client client = null;
+ try {
+client = getTabletServerConnection(tsi);
+boolean tserverMatch =
client.isRunningExternalCompaction(TraceUtil.traceInfo(),
+getContext().rpcCreds(), job.getExternalCompactionId(),
job.getExtent());
+if (tserverMatch) {
+
RUNNING.put(ExternalCompactionId.of(job.getExternalCompactionId()),
+ new RunningCompaction(job,
ExternalCompactionUtil.getHostPortString(hp),
+ tsi));
+ matchFound = true;
+}
+ } catch (TException e) {
+LOG.warn("Failed to notify tserver {}",
+tabletMetadata.getLocation().getHostAndPort(), e);
+ } finally {
+ThriftUtil.returnClient(client);
}
-} catch (TException e) {
- LOG.error(
- "Error from tserver {} while trying to
