(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit a61da25742eb4a3ab4860d71f24aeb156ad8ed25
Merge: 665484b029 90821b115c
Author: Keith Turner
AuthorDate: Mon Nov 20 21:18:15 2023 -0500
Merge branch 'main' into elasticity
.../accumulo/core/metrics/MetricsProducer.java | 144 ++---
1 file changed, 72 insertions(+), 72 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
index 021c045cc0,7aea9a7a6c..6bfc307fe6
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
@@@ -47,8 -47,8 +47,8 @@@ import io.micrometer.core.instrument.Me
*
* N/A
* N/A
- * {@link #METRICS_LOW_MEMORY}
+ * {@value #METRICS_LOW_MEMORY}
- * Guage
+ * Gauge
* reports 1 when process memory usage is above threshold, 0 when memory
is okay
*
*
@@@ -59,51 -59,9 +59,51 @@@
*
*
*
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUES}
+ * Gauge
+ *
+ *
+ *
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_LENGTH}
+ * Gauge
+ *
+ *
+ *
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_PRIORITY}
+ * Gauge
+ *
+ *
+ *
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_QUEUED}
+ * Gauge
+ *
+ *
+ *
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_DEQUEUED}
+ * Gauge
+ *
+ *
+ *
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_REJECTED}
+ * Gauge
+ *
+ *
+ *
* currentFateOps
* Gauge
- * {@link #METRICS_FATE_TOTAL_IN_PROGRESS}
+ * {@value #METRICS_FATE_TOTAL_IN_PROGRESS}
* Gauge
*
*
@@@ -286,9 -244,30 +286,9 @@@
*
*
*
- * activeMajCs
- * Gauge
- * {@value #METRICS_TSERVER_MAJC_RUNNING}
- * Gauge
- *
- *
- *
- * N/A
- * N/A
- * {@value #METRICS_TSERVER_MAJC_STUCK}
- * LongTaskTimer
- *
- *
- *
- * queuedMajCs
- * Gauge
- * {@value #METRICS_TSERVER_MAJC_QUEUED}
- * Gauge
- *
- *
- *
* activeMinCs
* Gauge
- * {@link #METRICS_TSERVER_MINC_RUNNING}
+ * {@value #METRICS_TSERVER_MINC_RUNNING}
* Gauge
*
*
@@@ -314,24 -293,9 +314,24 @@@
*
*
*
+ *
+ *
+ * {@link #METRICS_TSERVER_TABLETS_ONLINE_ONDEMAND}
+ * Gauge
+ * Represents the number of on-demand tablets that are online
+ *
+ *
+ *
+ *
+ * {@link #METRICS_TSERVER_TABLETS_ONDEMAND_UNLOADED_FOR_MEM}
+ * Gauge
+ * Represents the number of on-demand tablets that were unloaded due to
low memory
+ * condition
+ *
+ *
* N/A
* N/A
- * {@link #METRICS_TSERVER_TABLETS_LONG_ASSIGNMENTS}
+ * {@value #METRICS_TSERVER_TABLETS_LONG_ASSIGNMENTS}
* Gauge
*
*
(accumulo) branch elasticity updated (665484b029 -> a61da25742)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 665484b029 Restores column family validation in TabletMetadata (#3958) add e57df50bdd fixes bug with compaction preemption (#3961) add af53242538 Replaces java constant name with value in metrics javadocs (#3956) add 90821b115c Merge branch '2.1' new a61da25742 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/metrics/MetricsProducer.java | 144 ++--- 1 file changed, 72 insertions(+), 72 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 90821b115c90564004eb62625dab06864d4913be
Merge: b8089dd528 af53242538
Author: Keith Turner
AuthorDate: Mon Nov 20 21:13:32 2023 -0500
Merge branch '2.1'
.../accumulo/core/metrics/MetricsProducer.java | 150 ++---
.../accumulo/tserver/tablet/CompactableImpl.java | 11 +-
.../tablet/CompactableImplFileManagerTest.java | 34 +
3 files changed, 117 insertions(+), 78 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
index a2aa7d6a9a,1a8d618ce4..7aea9a7a6c
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
@@@ -47,14 -47,7 +47,14 @@@ import io.micrometer.core.instrument.Me
*
* N/A
* N/A
- * {@link #METRICS_LOW_MEMORY}
++ * {@value #METRICS_LOW_MEMORY}
+ * Guage
+ * reports 1 when process memory usage is above threshold, 0 when memory
is okay
+ *
+ *
+ * N/A
+ * N/A
- * {@link #METRICS_COMPACTOR_MAJC_STUCK}
+ * {@value #METRICS_COMPACTOR_MAJC_STUCK}
* LongTaskTimer
*
*
@@@ -431,20 -424,6 +431,20 @@@
* Counter
*
*
+ *
+ * N/A
+ * N/A
- * {@link #METRICS_SCAN_PAUSED_FOR_MEM}
++ * {@value #METRICS_SCAN_PAUSED_FOR_MEM}
+ * Counter
+ *
+ *
+ *
+ * N/A
+ * N/A
- * {@link #METRICS_SCAN_RETURN_FOR_MEM}
++ * {@value #METRICS_SCAN_RETURN_FOR_MEM}
+ * Counter
+ *
+ *
*
*
* {i|e}_{compactionServiceName}_{executor_name}_queued
@@@ -484,11 -456,40 +484,11 @@@
* Timer
*
*
- *
*
- * ReplicationQueue
- * Stat
- * {@value #METRICS_REPLICATION_QUEUE}
- * Timer
- *
- *
- *
- * ReplicationQueue10m
- * Quantiles
- * N/A
- * N/A
- *
- *
- *
- * filesPendingReplication
- * Stat
- * {@value #METRICS_REPLICATION_PENDING_FILES}
- * Gauge
*
- *
- *
- * maxReplicationThreads
- * Stat
- * {@value #METRICS_REPLICATION_THREADS}
- * Gauge
*
- * {@link #METRICS_MINC_PAUSED}
- *
- *
- * numPeers
- * Stat
- * {@value #METRICS_REPLICATION_PEERS}
- * Gauge
++ * {@value #METRICS_MINC_PAUSED}
+ * Counter
*
*
*
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index 8fe91dab5b,11722c100f..ce91f5d3ac
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@@ -872,11 -1081,10 +878,10 @@@ public class CompactableImpl implement
}
}
}
-
}
- static Collection asFileNames(Set files) {
-return Collections2.transform(files, StoredTabletFile::getFileName);
+ static Collection asMinimalString(Set files) {
+return Collections2.transform(files, StoredTabletFile::toMinimalString);
}
@Override
(accumulo) branch main updated (b8089dd528 -> 90821b115c)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from b8089dd528 Add a merge column marker to make merge idempotent (#3957) add e57df50bdd fixes bug with compaction preemption (#3961) add af53242538 Replaces java constant name with value in metrics javadocs (#3956) new 90821b115c Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/metrics/MetricsProducer.java | 150 ++--- .../accumulo/tserver/tablet/CompactableImpl.java | 11 +- .../tablet/CompactableImplFileManagerTest.java | 34 + 3 files changed, 117 insertions(+), 78 deletions(-)
(accumulo) branch 2.1 updated: Replaces java constant name with value in metrics javadocs (#3956)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new af53242538 Replaces java constant name with value in metrics javadocs
(#3956)
af53242538 is described below
commit af5324253861f2bd7ff46fddda98269fb867fcd5
Author: Keith Turner
AuthorDate: Mon Nov 20 18:09:01 2023 -0800
Replaces java constant name with value in metrics javadocs (#3956)
The metrics javadocs, which are referenced from the user manual, have a
table that currently contains java constant names. From the perspective
of a user they would probably prefer the value of the java constant,
which is currently cumbersome to obtain. This commit replaces the java
constant name with the values in the javadoc.
---
.../accumulo/core/metrics/MetricsProducer.java | 150 ++---
1 file changed, 75 insertions(+), 75 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
index c76a682f1b..1a8d618ce4 100644
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
@@ -47,21 +47,21 @@ import io.micrometer.core.instrument.MeterRegistry;
*
* N/A
* N/A
- * {@link #METRICS_COMPACTOR_MAJC_STUCK}
+ * {@value #METRICS_COMPACTOR_MAJC_STUCK}
* LongTaskTimer
*
*
*
* currentFateOps
* Gauge
- * {@link #METRICS_FATE_TOTAL_IN_PROGRESS}
+ * {@value #METRICS_FATE_TOTAL_IN_PROGRESS}
* Gauge
*
*
*
* FateTxOpType_{name}
* Gauge
- * {@link #METRICS_FATE_TYPE_IN_PROGRESS}
+ * {@value #METRICS_FATE_TYPE_IN_PROGRESS}
* Gauge
* Previously there was a metric per operation type with the count of
in-progress transactions
* of that type. Now there is one metric and the type is in the tag
op.type
@@ -69,56 +69,56 @@ import io.micrometer.core.instrument.MeterRegistry;
*
* totalFateOps
* Gauge
- * {@link #METRICS_FATE_OPS_ACTIVITY}
+ * {@value #METRICS_FATE_OPS_ACTIVITY}
* Gauge
*
*
*
* totalZkConnErrors
* Gauge
- * {@link #METRICS_FATE_ERRORS}
+ * {@value #METRICS_FATE_ERRORS}
* Gauge
*
*
*
* FateTxState_NEW
* Gauge
- * {@link #METRICS_FATE_TX}
+ * {@value #METRICS_FATE_TX}
* Gauge
* The state is now in a tag: state=new
*
*
* FateTxState_IN_PROGRESS
* Gauge
- * {@link #METRICS_FATE_TX}
+ * {@value #METRICS_FATE_TX}
* Gauge
* The state is now in a tag: state=in.progress
*
*
* FateTxState_FAILED_IN_PROGRESS
* Gauge
- * {@link #METRICS_FATE_TX}
+ * {@value #METRICS_FATE_TX}
* Gauge
* The state is now in a tag: state=failed.in.progress
*
*
* FateTxState_FAILED
* Gauge
- * {@link #METRICS_FATE_TX}
+ * {@value #METRICS_FATE_TX}
* Gauge
* The state is now in a tag: state=failed
*
*
* FateTxState_SUCCESSFUL
* Gauge
- * {@link #METRICS_FATE_TX}
+ * {@value #METRICS_FATE_TX}
* Gauge
* The state is now in a tag: state=successful
*
*
* FateTxState_UNKNOWN
* Gauge
- * {@link #METRICS_FATE_TX}
+ * {@value #METRICS_FATE_TX}
* Gauge
* The state is now in a tag: state=unknown
*
@@ -126,98 +126,98 @@ import io.micrometer.core.instrument.MeterRegistry;
*
* AccGcStarted
* Gauge
- * {@link #METRICS_GC_STARTED}
+ * {@value #METRICS_GC_STARTED}
* Gauge
*
*
*
* AccGcFinished
* Gauge
- * {@link #METRICS_GC_FINISHED}
+ * {@value #METRICS_GC_FINISHED}
* Gauge
*
*
*
* AccGcCandidates
* Gauge
- * {@link #METRICS_GC_CANDIDATES}
+ * {@value #METRICS_GC_CANDIDATES}
* Gauge
*
*
*
* AccGcInUse
* Gauge
- * {@link #METRICS_GC_IN_USE}
+ * {@value #METRICS_GC_IN_USE}
* Gauge
*
*
*
* AccGcDeleted
* Gauge
- * {@link #METRICS_GC_DELETED}
+ * {@value #METRICS_GC_DELETED}
* Gauge
*
*
*
* AccGcErrors
* Gauge
- * {@link #METRICS_GC_ERRORS}
+ * {@value #METRICS_GC_ERRORS}
* Gauge
*
*
*
* AccGcWalStarted
* Gauge
- * {@link #METRICS_GC_WAL_STARTED}
+ * {@value #METRICS_GC_WAL_STARTED}
* Gauge
*
*
*
* AccGcWalFinished
* Gauge
- * {@link #METRICS_GC_WAL_FINISHED}
+ * {@value #METRICS_GC_WAL_FINISHED}
* Gauge
*
*
*
* AccGcWalCandidates
* Gauge
- * {@link #METRICS_GC_WAL_CANDIDATES}
+ * {@value #METRICS_GC_WAL_CANDIDATES}
* Gauge
*
*
*
* AccGcWalInUse
* Gauge
- * {@link #METRICS_GC_WAL_IN_USE}
+ * {@value #METRICS_GC_WAL_IN_USE}
* Gauge
*
*
*
* AccGcWalDeleted
* Gauge
- * {@link #METRICS_GC_WAL_DELETED}
+ * {@value #METRICS_GC_WAL_DELETED}
* Gauge
*
*
*
* AccGcWalErrors
* Gauge
- * {@link #METRICS_GC_WAL_ERRORS}
+ * {@value #METRICS_GC_WAL_ERRORS}
* Gauge
*
*
*
* AccGcPosOpDuration
* Gauge
- * {@link #METRICS_GC_POST_OP_DURATION}
+ * {@value
(accumulo) branch 2.1 updated: fixes bug with compaction preemption (#3961)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new e57df50bdd fixes bug with compaction preemption (#3961)
e57df50bdd is described below
commit e57df50bdd940e1115cf2f55032ca3e7c2421f2b
Author: Keith Turner
AuthorDate: Mon Nov 20 18:08:33 2023 -0800
fixes bug with compaction preemption (#3961)
User compactions can preempt selector compaction under certain
conditions. There was a bug in the code where a user compaction could
preempt a selection compaction that was in the middle of selecting its
files. The way the code is structured doing this causes an exception.
This change makes the preemption wait until file selection is complete.
---
.../accumulo/tserver/tablet/CompactableImpl.java | 11 +--
.../tablet/CompactableImplFileManagerTest.java | 34 ++
2 files changed, 42 insertions(+), 3 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index 1a820bee5b..11722c100f 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@ -81,6 +81,7 @@ import org.apache.zookeeper.KeeperException.NoNodeException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Suppliers;
import com.google.common.collect.Collections2;
@@ -231,6 +232,11 @@ public class CompactableImpl implements Compactable {
return selectKind;
}
+@VisibleForTesting
+Set getSelectedFiles() {
+ return Set.copyOf(selectedFiles);
+}
+
SelectedInfo getReservedInfo() {
Preconditions.checkState(selectStatus == FileSelectionStatus.RESERVED);
return new SelectedInfo(initiallySelectedAll, selectedFiles, selectKind);
@@ -245,7 +251,8 @@ public class CompactableImpl implements Compactable {
Preconditions.checkArgument(kind == CompactionKind.SELECTOR || kind ==
CompactionKind.USER);
if (selectStatus == FileSelectionStatus.NOT_ACTIVE || (kind ==
CompactionKind.USER
- && selectKind == CompactionKind.SELECTOR &&
noneRunning(CompactionKind.SELECTOR))) {
+ && selectKind == CompactionKind.SELECTOR &&
noneRunning(CompactionKind.SELECTOR)
+ && selectStatus != FileSelectionStatus.SELECTING)) {
selectStatus = FileSelectionStatus.NEW;
selectKind = kind;
selectedFiles.clear();
@@ -1065,7 +1072,6 @@ public class CompactableImpl implements Compactable {
manager.compactableChanged(this);
}
-
} catch (Exception e) {
log.error("Failed to select user compaction files {}", getExtent(), e);
} finally {
@@ -1075,7 +1081,6 @@ public class CompactableImpl implements Compactable {
}
}
}
-
}
static Collection asFileNames(Set files) {
diff --git
a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/CompactableImplFileManagerTest.java
b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/CompactableImplFileManagerTest.java
index 971ee51e31..8214823be3 100644
---
a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/CompactableImplFileManagerTest.java
+++
b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/CompactableImplFileManagerTest.java
@@ -288,6 +288,40 @@ public class CompactableImplFileManagerTest {
assertEquals(FileSelectionStatus.SELECTED, fileMgr.getSelectionStatus());
}
+ @Test
+ public void testUserCompactionPreemptsSelectorCompaction() {
+TestFileManager fileMgr = new TestFileManager();
+
+assertTrue(fileMgr.initiateSelection(SELECTOR));
+assertEquals(SELECTOR, fileMgr.getSelectionKind());
+assertTrue(fileMgr.beginSelection());
+// USER compaction should not be able to preempt while in the middle of
selecting files
+assertFalse(fileMgr.initiateSelection(USER));
+assertEquals(SELECTOR, fileMgr.getSelectionKind());
+fileMgr.finishSelection(newFiles("F0.rf", "F1.rf", "F2.rf"),
false);
+// check state is as expected after finishing selection
+assertEquals(SELECTOR, fileMgr.getSelectionKind());
+assertEquals(FileSelectionStatus.SELECTED, fileMgr.getSelectionStatus());
+assertFalse(fileMgr.getSelectedFiles().isEmpty());
+
+// USER compaction should not be able to preempt when there are running
compactions.
+fileMgr.running.add(SELECTOR);
+assertFalse(fileMgr.initiateSelection(USER));
+// check state is as expected
+assertEquals(SELECTOR, fileMgr.getSelectionKind());
+
