(accumulo) branch elasticity updated: Refresh the upgrade code (#4003)
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 21241b18d8 Refresh the upgrade code (#4003)
21241b18d8 is described below
commit 21241b18d86500cda6c1c584f3ec3ee9bb9cf0c5
Author: Christopher Tubbs
AuthorDate: Thu Nov 30 18:33:13 2023 -0500
Refresh the upgrade code (#4003)
Align the upgrade code to match the main branch conventions/style, but
keep it disabled for now, until it's working on the elasticity branch.
---
.../java/org/apache/accumulo/server/AccumuloDataVersion.java | 3 ++-
.../java/org/apache/accumulo/server/ServerContextTest.java | 10 ++
.../apache/accumulo/manager/upgrade/UpgradeCoordinator.java| 8 ++--
3 files changed, 10 insertions(+), 11 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
b/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
index ee700c4208..224cbde6af 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
@@ -83,7 +83,8 @@ public class AccumuloDataVersion {
}
// ELASTICITY_TODO get upgrade working
- // public static final Set CAN_RUN =
Set.of(ROOT_TABLET_META_CHANGES, CURRENT_VERSION);
+ // public static final Set CAN_RUN =
Set.of(ROOT_TABLET_META_CHANGES,
+ // REMOVE_DEPRECATIONS_FOR_VERSION_3, METADATA_FILE_JSON_ENCODING,
CURRENT_VERSION);
public static final Set CAN_RUN = Set.of(CURRENT_VERSION);
/**
diff --git
a/server/base/src/test/java/org/apache/accumulo/server/ServerContextTest.java
b/server/base/src/test/java/org/apache/accumulo/server/ServerContextTest.java
index 7d2535579c..d5909ecf99 100644
---
a/server/base/src/test/java/org/apache/accumulo/server/ServerContextTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/ServerContextTest.java
@@ -133,15 +133,9 @@ public class ServerContextTest {
// ensure upgrades fail with older, unsupported versions, but pass with
supported versions
@Test
public void testCanRun() {
-// ensure this fails with older versions; the oldest supported version is
hard-coded here
-// to ensure we don't unintentionally break upgrade support; changing this
should be a conscious
-// decision and this check will ensure we don't overlook it
-
-final int oldestSupported =
AccumuloDataVersion.ONDEMAND_TABLETS_FOR_VERSION_4;
-// ELASTICITY_TODO basically disable check until upgrade to 3.1 is
supported. Should be:
-// final int oldestSupported =
AccumuloDataVersion.oldestUpgradeableVersion();
+final int oldestSupported = AccumuloDataVersion.oldestUpgradeableVersion();
+// ELASTICITY_TODO basically disable check until upgrade is working.
Should be:
// assertEquals(10, oldestSupported); // make sure it hasn't changed
accidentally
-
final int currentVersion = AccumuloDataVersion.get();
IntConsumer shouldPass = ServerContext::ensureDataVersionCompatible;
IntConsumer shouldFail = v -> assertThrows(IllegalStateException.class,
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java
index 94a81eaaf9..ffd6c4fba1 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java
@@ -18,6 +18,8 @@
*/
package org.apache.accumulo.manager.upgrade;
+import static
org.apache.accumulo.server.AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+import static
org.apache.accumulo.server.AccumuloDataVersion.REMOVE_DEPRECATIONS_FOR_VERSION_3;
import static
org.apache.accumulo.server.AccumuloDataVersion.ROOT_TABLET_META_CHANGES;
import java.io.IOException;
@@ -112,8 +114,10 @@ public class UpgradeCoordinator {
private int currentVersion;
// map of "current version" -> upgrader to next version.
// Sorted so upgrades execute in order from the oldest supported data
version to current
- private final Map upgraders = Collections
- .unmodifiableMap(new TreeMap<>(Map.of(ROOT_TABLET_META_CHANGES, new
Upgrader10to11(;
+ private final Map upgraders = Collections.unmodifiableMap(new TreeMap<>(
+ Map.of(ROOT_TABLET_META_CHANGES, new Upgrader10to11(),
REMOVE_DEPRECATIONS_FOR_VERSION_3,
+ new Upgrader11to12(), METADATA_FILE_JSON_ENCODING, new
Upgrader12to13(;
private volatile UpgradeStatus status;
(accumulo) branch elasticity updated (afe9a8a87e -> 5c420f42eb)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from afe9a8a87e Add resourceGroup as a tag in the emitted metrics (#3992) add 751a110efe fixes bug with bulk import RPC failure and retry (#4000) add 6abb607d12 Merge branch '2.1' new 5c420f42eb Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../apache/accumulo/test/functional/BulkNewIT.java | 135 + 1 file changed, 135 insertions(+)
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 5c420f42ebfb91c33cc1da4080d61966f6cb0ad0
Merge: afe9a8a87e 6abb607d12
Author: Keith Turner
AuthorDate: Thu Nov 30 16:55:06 2023 -0500
Merge branch 'main' into elasticity
.../apache/accumulo/test/functional/BulkNewIT.java | 135 +
1 file changed, 135 insertions(+)
diff --cc test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
index 3e63bed845,6a0448749c..bbdb091fd3
--- a/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
@@@ -22,6 -23,8 +23,7 @@@ import static org.apache.accumulo.core.
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOADED;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertFalse;
+ import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@@ -48,24 -53,28 +51,32 @@@ import java.util.stream.Collectors
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
+ import org.apache.accumulo.core.client.AccumuloSecurityException;
+ import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
+ import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.client.admin.CompactionConfig;
import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.client.admin.TimeType;
+import org.apache.accumulo.core.clientImpl.ClientContext;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.LoadPlan;
import org.apache.accumulo.core.data.LoadPlan.RangeType;
+ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
+ import org.apache.accumulo.core.data.constraints.Constraint;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVWriter;
import org.apache.accumulo.core.file.rfile.RFile;
+ import org.apache.accumulo.core.metadata.MetadataTable;
+ import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.UnreferencedTabletFile;
+ import org.apache.accumulo.core.metadata.schema.MetadataSchema;
+import org.apache.accumulo.core.metadata.schema.MetadataTime;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.security.Authorizations;
@@@ -511,29 -521,43 +528,56 @@@ public class BulkNewIT extends SharedMi
}
}
+ @Test
+ public void testManyFiles() throws Exception {
+try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
+ String dir = getDir("/testBulkFile-");
+ FileSystem fs = getCluster().getFileSystem();
+ fs.mkdirs(new Path(dir));
+
+ addSplits(c, tableName, "5000");
+
+ for (int i = 0; i < 100; i++) {
+writeData(dir + "/f" + i + ".", aconf, i * 100, (i + 1) * 100 - 1);
+ }
+
+ c.tableOperations().importDirectory(dir).to(tableName).load();
+
+ verifyData(c, tableName, 0, 100 * 100 - 1, false);
+
+ c.tableOperations().compact(tableName, new
CompactionConfig().setWait(true));
+
+ verifyData(c, tableName, 0, 100 * 100 - 1, false);
+}
+ }
+
+ @Test
+ public void testExceptionInMetadataUpdate() throws Exception {
+ try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
+
- // after setting this up, bulk imports should never succeed on a tablet
server
++ // after setting this up, bulk imports should fail
+ setupBulkConstraint(getPrincipal(), c);
+
+ String dir = getDir("/testExceptionInMetadataUpdate-");
+
- String h1 = writeData(dir + "/f1.", aconf, 0, 333);
-
- var executor = Executors.newSingleThreadExecutor();
- // With the constraint configured that makes tservers throw an
exception on bulk import, the
- // bulk import should never succeed. So run the bulk import in another
thread.
- var future = executor.submit(() -> {
-c.tableOperations().importDirectory(dir).to(tableName).load();
-return null;
- });
-
-
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 6abb607d12216fd38057df66154df02bdb854f0c
Merge: a03d364632 751a110efe
Author: Keith Turner
AuthorDate: Thu Nov 30 16:19:24 2023 -0500
Merge branch '2.1'
.../org/apache/accumulo/tserver/tablet/Tablet.java | 10 +-
.../apache/accumulo/test/functional/BulkNewIT.java | 147 +
2 files changed, 152 insertions(+), 5 deletions(-)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index fc373ae30c,3f7eebe95f..1b6ba2f3cf
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@@ -1770,9 -1787,14 +1770,14 @@@ public class Tablet extends TabletBase
try {
tabletServer.updateBulkImportState(files, BulkImportState.LOADING);
- var storedTabletFile = getDatafileManager().importMapFiles(tid,
entries, setTime);
- lastMapFileImportTime = System.currentTimeMillis();
+ getDatafileManager().importDataFiles(tid, entries, setTime);
+ lastDataFileImportTime = System.currentTimeMillis();
+ synchronized (this) {
+ // only mark the bulk import a success if no exception was thrown
+ bulkImported.computeIfAbsent(tid, k -> new
ArrayList<>()).addAll(fileMap.keySet());
+ }
+
if (isSplitPossible()) {
getTabletServer().executeSplit(this);
} else {
diff --cc test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
index 5280d279dc,a8aebd2e25..6a0448749c
--- a/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
@@@ -61,7 -71,9 +71,10 @@@ import org.apache.accumulo.core.data.co
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVWriter;
import org.apache.accumulo.core.file.rfile.RFile;
+ import org.apache.accumulo.core.metadata.MetadataTable;
+ import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.UnreferencedTabletFile;
+ import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.security.Authorizations;
(accumulo) branch main updated (a03d364632 -> 6abb607d12)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from a03d364632 Merge branch '2.1' add 751a110efe fixes bug with bulk import RPC failure and retry (#4000) new 6abb607d12 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/tserver/tablet/Tablet.java | 10 +- .../apache/accumulo/test/functional/BulkNewIT.java | 147 + 2 files changed, 152 insertions(+), 5 deletions(-)
(accumulo) branch 2.1 updated: fixes bug with bulk import RPC failure and retry (#4000)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 751a110efe fixes bug with bulk import RPC failure and retry (#4000)
751a110efe is described below
commit 751a110efe6ecf43c0aa6662160fa3766d881936
Author: Keith Turner
AuthorDate: Thu Nov 30 16:13:51 2023 -0500
fixes bug with bulk import RPC failure and retry (#4000)
The bulk import code had a bug where if an exception was thrown in a
certain place in the tablet code that it would mark the bulk import as
complete. After the exception, other code would retry and see the bulk
import was complete even though it was not. This commit changes the
behavior to only note success when there is no exception. In addition
test were added that recreated this bug. These test set a constraint on
the metadata table that cause bulk import writes to fail.
---
.../org/apache/accumulo/tserver/tablet/Tablet.java | 10 +-
.../apache/accumulo/test/functional/BulkNewIT.java | 147 +
.../apache/accumulo/test/functional/BulkOldIT.java | 51 +++
3 files changed, 203 insertions(+), 5 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 175f3a1f27..3f7eebe95f 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -1790,6 +1790,11 @@ public class Tablet extends TabletBase {
var storedTabletFile = getDatafileManager().importMapFiles(tid, entries,
setTime);
lastMapFileImportTime = System.currentTimeMillis();
+ synchronized (this) {
+// only mark the bulk import a success if no exception was thrown
+bulkImported.computeIfAbsent(tid, k -> new
ArrayList<>()).addAll(fileMap.keySet());
+ }
+
if (isSplitPossible()) {
getTabletServer().executeSplit(this);
} else {
@@ -1804,11 +1809,6 @@ public class Tablet extends TabletBase {
"Likely bug in code, always expect to remove something. Please
open an Accumulo issue.");
}
-try {
- bulkImported.computeIfAbsent(tid, k -> new
ArrayList<>()).addAll(fileMap.keySet());
-} catch (Exception ex) {
- log.info(ex.toString(), ex);
-}
tabletServer.removeBulkImportState(files);
}
}
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
index afa0afc677..a8aebd2e25 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
@@ -18,10 +18,13 @@
*/
package org.apache.accumulo.test.functional;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.FILES;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOADED;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -37,18 +40,23 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
import java.util.stream.Collectors;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.TableNotFoundException;
import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.client.admin.TimeType;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
@@ -56,20 +64,30 @@ import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.LoadPlan;
import org.apache.accumulo.core.data.LoadPlan.RangeType;
+import org.apache.accumulo.core.data.Mutation;
import
(accumulo) branch elasticity updated: Add resourceGroup as a tag in the emitted metrics (#3992)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new afe9a8a87e Add resourceGroup as a tag in the emitted metrics (#3992)
afe9a8a87e is described below
commit afe9a8a87ea665ca89056208b4f8f0b91f8ce9b7
Author: Dave Marion
AuthorDate: Thu Nov 30 16:08:16 2023 -0500
Add resourceGroup as a tag in the emitted metrics (#3992)
Related to #3991
---
.../apache/accumulo/core/metrics/MetricsUtil.java| 20 +++-
.../org/apache/accumulo/compactor/Compactor.java | 2 +-
.../apache/accumulo/gc/SimpleGarbageCollector.java | 2 +-
.../java/org/apache/accumulo/manager/Manager.java| 2 +-
.../java/org/apache/accumulo/tserver/ScanServer.java | 2 +-
.../org/apache/accumulo/tserver/TabletServer.java| 2 +-
.../org/apache/accumulo/test/metrics/MetricsIT.java | 5 -
7 files changed, 20 insertions(+), 15 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java
b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java
index 57bd50bb3b..cb9b9481c3 100644
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java
@@ -53,19 +53,21 @@ public class MetricsUtil {
private static Pattern camelCasePattern = Pattern.compile("[a-z][A-Z][a-z]");
public static void initializeMetrics(final AccumuloConfiguration conf, final
String appName,
- final HostAndPort address, final String instanceName) throws
ClassNotFoundException,
- InstantiationException, IllegalAccessException, IllegalArgumentException,
- InvocationTargetException, NoSuchMethodException, SecurityException {
+ final HostAndPort address, final String instanceName, final String
resourceGroup)
+ throws ClassNotFoundException, InstantiationException,
IllegalAccessException,
+ IllegalArgumentException, InvocationTargetException,
NoSuchMethodException,
+ SecurityException {
initializeMetrics(conf.getBoolean(Property.GENERAL_MICROMETER_ENABLED),
conf.getBoolean(Property.GENERAL_MICROMETER_JVM_METRICS_ENABLED),
-conf.get(Property.GENERAL_MICROMETER_FACTORY), appName, address,
instanceName);
+conf.get(Property.GENERAL_MICROMETER_FACTORY), appName, address,
instanceName,
+resourceGroup);
}
private static void initializeMetrics(boolean enabled, boolean
jvmMetricsEnabled,
- String factoryClass, String appName, HostAndPort address, String
instanceName)
- throws ClassNotFoundException, InstantiationException,
IllegalAccessException,
- IllegalArgumentException, InvocationTargetException,
NoSuchMethodException,
- SecurityException {
+ String factoryClass, String appName, HostAndPort address, String
instanceName,
+ String resourceGroup) throws ClassNotFoundException,
InstantiationException,
+ IllegalAccessException, IllegalArgumentException,
InvocationTargetException,
+ NoSuchMethodException, SecurityException {
LOG.info("initializing metrics, enabled:{}, class:{}", enabled,
factoryClass);
@@ -80,7 +82,7 @@ public class MetricsUtil {
List tags = new ArrayList<>();
tags.add(Tag.of("instance.name", instanceName));
tags.add(Tag.of("process.name", processName));
-
+ tags.add(Tag.of("resource.group", resourceGroup));
if (address != null) {
if (!address.getHost().isEmpty()) {
tags.add(Tag.of("host", address.getHost()));
diff --git
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index 73ac3e2e83..9e3c81504c 100644
---
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@ -613,7 +613,7 @@ public class Compactor extends AbstractServer implements
MetricsProducer, Compac
try {
MetricsUtil.initializeMetrics(getContext().getConfiguration(),
this.applicationName,
- clientAddress, getContext().getInstanceName());
+ clientAddress, getContext().getInstanceName(),
this.getResourceGroup());
pausedMetrics = new PausedCompactionMetrics();
MetricsUtil.initializeProducers(this, pausedMetrics);
} catch (ClassNotFoundException | InstantiationException |
IllegalAccessException
diff --git
a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index e52d63291a..6bfb5b07c5 100644
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@ -158,7
(accumulo) branch elasticity updated (fc2a2e7dd3 -> 4346e744a7)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from fc2a2e7dd3 Partially revert 1414641 to resurrect 2.1 upgrade code (#3999) add a47ee0b77d Polish up some code from recent changes (#3979) add e34f7fe1ce Add instance name tag to metrics (#3998) add a03d364632 Merge branch '2.1' add 4346e744a7 Merge branch 'main' into elasticity No new revisions were added by this update. Summary of changes: .../apache/accumulo/core/conf/PropertyType.java| 2 +- .../core/metadata/schema/MetadataSchema.java | 10 ++ .../schema/UpgraderDeprecatedConstants.java| 40 -- .../apache/accumulo/core/metrics/MetricsUtil.java | 16 + .../accumulo/core/tabletserver/log/LogEntry.java | 4 +-- .../compaction/DefaultCompactionPlannerTest.java | 6 ++-- .../accumulo/server/AccumuloDataVersion.java | 12 --- .../server/constraints/MetadataConstraints.java| 10 -- .../apache/accumulo/server/ServerContextTest.java | 4 ++- .../org/apache/accumulo/compactor/Compactor.java | 2 +- .../apache/accumulo/gc/SimpleGarbageCollector.java | 3 +- .../java/org/apache/accumulo/manager/Manager.java | 2 +- .../accumulo/manager/upgrade/Upgrader11to12.java | 14 .../manager/upgrade/Upgrader11to12Test.java| 7 ++-- .../org/apache/accumulo/tserver/ScanServer.java| 2 +- .../org/apache/accumulo/tserver/TabletServer.java | 4 +-- .../apache/accumulo/test/metrics/MetricsIT.java| 2 ++ 17 files changed, 64 insertions(+), 76 deletions(-) delete mode 100644 core/src/main/java/org/apache/accumulo/core/metadata/schema/UpgraderDeprecatedConstants.java
(accumulo) branch elasticity updated: Partially revert 1414641 to resurrect 2.1 upgrade code (#3999)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new fc2a2e7dd3 Partially revert 1414641 to resurrect 2.1 upgrade code
(#3999)
fc2a2e7dd3 is described below
commit fc2a2e7dd32ac52da3e861c2f05b126b3b07d7b0
Author: Dave Marion
AuthorDate: Thu Nov 30 15:24:11 2023 -0500
Partially revert 1414641 to resurrect 2.1 upgrade code (#3999)
---
.../accumulo/server/AccumuloDataVersion.java | 12 +-
.../apache/accumulo/server/ServerContextTest.java | 2 +-
.../manager/upgrade/UpgradeCoordinator.java| 6 +-
.../accumulo/manager/upgrade/Upgrader10to11.java | 268 +
.../manager/upgrade/Upgrader10to11Test.java| 192 +++
5 files changed, 474 insertions(+), 6 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
b/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
index a96721988e..9f459ea599 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
@@ -37,7 +37,7 @@ import java.util.Set;
public class AccumuloDataVersion {
/**
- * version (12) reflects On-Demand tablets starting with 4.0
+ * version (13) reflects On-Demand tablets starting with 4.0
*/
public static final int ONDEMAND_TABLETS_FOR_VERSION_4 = 13;
@@ -52,11 +52,16 @@ public class AccumuloDataVersion {
*/
public static final int REMOVE_DEPRECATIONS_FOR_VERSION_3 = 11;
+ /**
+ * version (10) reflects changes to how root tablet metadata is serialized
in zookeeper starting
+ * with 2.1. See {@link
org.apache.accumulo.core.metadata.schema.RootTabletMetadata}.
+ */
+ public static final int ROOT_TABLET_META_CHANGES = 10;
+
/**
* Historic data versions
*
*
- * version (10) Changes to how root tablet metadata is serialized in
zookeeper in 2.1.0
* version (9) RFiles and wal crypto serialization changes. RFile
summary data in 2.0.0
* version (8) RFile index (ACCUMULO-1124) and wal tracking in ZK in
1.8.0
* version (7) also reflects the addition of a replication table in 1.7.0
@@ -78,6 +83,7 @@ public class AccumuloDataVersion {
}
// ELASTICITY_TODO get upgrade working
+ // public static final Set CAN_RUN =
Set.of(ROOT_TABLET_META_CHANGES, CURRENT_VERSION);
public static final Set CAN_RUN = Set.of(CURRENT_VERSION);
/**
@@ -99,6 +105,8 @@ public class AccumuloDataVersion {
private static String dataVersionToReleaseName(final int version) {
switch (version) {
+ case ROOT_TABLET_META_CHANGES:
+return "2.1.0";
case REMOVE_DEPRECATIONS_FOR_VERSION_3:
return "3.0.0";
case METADATA_FILE_JSON_ENCODING:
diff --git
a/server/base/src/test/java/org/apache/accumulo/server/ServerContextTest.java
b/server/base/src/test/java/org/apache/accumulo/server/ServerContextTest.java
index 7694b34166..f86127ea9b 100644
---
a/server/base/src/test/java/org/apache/accumulo/server/ServerContextTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/ServerContextTest.java
@@ -138,7 +138,7 @@ public class ServerContextTest {
final int oldestSupported =
AccumuloDataVersion.ONDEMAND_TABLETS_FOR_VERSION_4;
// ELASTICITY_TODO basically disable check until upgrade to 3.1 is
supported. Should be:
-// final int oldestSupported =
AccumuloDataVersion.METADATA_FILE_JSON_ENCODING;
+// final int oldestSupported =
AccumuloDataVersion.ROOT_TABLET_META_CHANGES;
final int currentVersion = AccumuloDataVersion.get();
IntConsumer shouldPass = ServerContext::ensureDataVersionCompatible;
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java
index dedda83872..94a81eaaf9 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java
@@ -18,7 +18,7 @@
*/
package org.apache.accumulo.manager.upgrade;
-import static
org.apache.accumulo.server.AccumuloDataVersion.REMOVE_DEPRECATIONS_FOR_VERSION_3;
+import static
org.apache.accumulo.server.AccumuloDataVersion.ROOT_TABLET_META_CHANGES;
import java.io.IOException;
import java.util.Collections;
@@ -112,8 +112,8 @@ public class UpgradeCoordinator {
private int currentVersion;
// map of "current version" -> upgrader to next version.
// Sorted so upgrades execute in order from the oldest supported data
version to current
- private final Map upgraders = Collections.unmodifiableMap(
- new
(accumulo) branch main updated (a47ee0b77d -> a03d364632)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from a47ee0b77d Polish up some code from recent changes (#3979) add e34f7fe1ce Add instance name tag to metrics (#3998) new a03d364632 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/core/metrics/MetricsUtil.java| 16 +--- .../accumulo/coordinator/CompactionCoordinator.java | 2 +- .../java/org/apache/accumulo/compactor/Compactor.java| 2 +- .../org/apache/accumulo/gc/SimpleGarbageCollector.java | 3 ++- .../main/java/org/apache/accumulo/manager/Manager.java | 2 +- .../java/org/apache/accumulo/tserver/ScanServer.java | 2 +- .../java/org/apache/accumulo/tserver/TabletServer.java | 4 ++-- .../java/org/apache/accumulo/test/metrics/MetricsIT.java | 2 ++ 8 files changed, 19 insertions(+), 14 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit a03d36463257f1a911812f76015b22da2f33110a
Merge: a47ee0b77d e34f7fe1ce
Author: Dave Marion
AuthorDate: Thu Nov 30 18:11:37 2023 +
Merge branch '2.1'
.../org/apache/accumulo/core/metrics/MetricsUtil.java| 16 +---
.../accumulo/coordinator/CompactionCoordinator.java | 2 +-
.../java/org/apache/accumulo/compactor/Compactor.java| 2 +-
.../org/apache/accumulo/gc/SimpleGarbageCollector.java | 3 ++-
.../main/java/org/apache/accumulo/manager/Manager.java | 2 +-
.../java/org/apache/accumulo/tserver/ScanServer.java | 2 +-
.../java/org/apache/accumulo/tserver/TabletServer.java | 4 ++--
.../java/org/apache/accumulo/test/metrics/MetricsIT.java | 2 ++
8 files changed, 19 insertions(+), 14 deletions(-)
diff --cc
server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index ef84295540,88df289a56..cb7c4731bb
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@@ -266,8 -268,7 +266,8 @@@ public class CompactionCoordinator exte
try {
MetricsUtil.initializeMetrics(getContext().getConfiguration(),
this.applicationName,
- clientAddress);
+ clientAddress, getContext().getInstanceName());
+ MetricsUtil.initializeProducers(this);
} catch (ClassNotFoundException | InstantiationException |
IllegalAccessException
| IllegalArgumentException | InvocationTargetException |
NoSuchMethodException
| SecurityException e1) {
diff --cc
server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index fc7d6a0234,9662757d3d..682437d298
---
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@@ -608,9 -616,7 +608,9 @@@ public class Compactor extends Abstract
try {
MetricsUtil.initializeMetrics(getContext().getConfiguration(),
this.applicationName,
- clientAddress);
+ clientAddress, getContext().getInstanceName());
+ pausedMetrics = new PausedCompactionMetrics();
+ MetricsUtil.initializeProducers(this, pausedMetrics);
} catch (ClassNotFoundException | InstantiationException |
IllegalAccessException
| IllegalArgumentException | InvocationTargetException |
NoSuchMethodException
| SecurityException e1) {
diff --cc
server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 01198752d7,0c22c4b50a..7152016d73
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@@ -157,8 -168,9 +157,9 @@@ public class SimpleGarbageCollector ext
}
try {
- MetricsUtil.initializeMetrics(getContext().getConfiguration(),
this.applicationName, address);
+ MetricsUtil.initializeMetrics(getContext().getConfiguration(),
this.applicationName, address,
+ getContext().getInstanceName());
- MetricsUtil.initializeProducers(new GcMetrics(this));
+ MetricsUtil.initializeProducers(this, new GcMetrics(this));
} catch (ClassNotFoundException | InstantiationException |
IllegalAccessException
| IllegalArgumentException | InvocationTargetException |
NoSuchMethodException
| SecurityException e1) {
diff --cc server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index b75c8601c4,9d71e542f9..7d3806840f
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@@ -1103,9 -1099,8 +1103,9 @@@ public class Manager extends AbstractSe
try {
MetricsUtil.initializeMetrics(getContext().getConfiguration(),
this.applicationName,
- sa.getAddress());
+ sa.getAddress(), getContext().getInstanceName());
- ManagerMetrics.init(getConfiguration(), this);
+ ManagerMetrics mm = new ManagerMetrics(getConfiguration(), this);
+ MetricsUtil.initializeProducers(this, mm);
} catch (ClassNotFoundException | InstantiationException |
IllegalAccessException
| IllegalArgumentException | InvocationTargetException |
NoSuchMethodException
| SecurityException e1) {
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
index 8868065aa4,a34c2b1adc..fe227959f7
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
@@@ -370,9 -372,7 +370,9 @@@ public class ScanServer extends Abstrac
(accumulo) branch 2.1 updated: Add instance name tag to metrics (#3998)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new e34f7fe1ce Add instance name tag to metrics (#3998)
e34f7fe1ce is described below
commit e34f7fe1ce5c04f39e63782e3fdb7394a7f7204c
Author: Dave Marion
AuthorDate: Thu Nov 30 12:53:37 2023 -0500
Add instance name tag to metrics (#3998)
---
.../org/apache/accumulo/core/metrics/MetricsUtil.java| 16 +---
.../accumulo/coordinator/CompactionCoordinator.java | 2 +-
.../java/org/apache/accumulo/compactor/Compactor.java| 2 +-
.../org/apache/accumulo/gc/SimpleGarbageCollector.java | 3 ++-
.../main/java/org/apache/accumulo/manager/Manager.java | 2 +-
.../java/org/apache/accumulo/tserver/ScanServer.java | 2 +-
.../java/org/apache/accumulo/tserver/TabletServer.java | 4 ++--
.../java/org/apache/accumulo/test/metrics/MetricsIT.java | 2 ++
8 files changed, 19 insertions(+), 14 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java
b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java
index b7e3ff5ae4..b84d1fc0f5 100644
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsUtil.java
@@ -53,18 +53,19 @@ public class MetricsUtil {
private static List commonTags;
public static void initializeMetrics(final AccumuloConfiguration conf, final
String appName,
- final HostAndPort address) throws ClassNotFoundException,
InstantiationException,
- IllegalAccessException, IllegalArgumentException,
InvocationTargetException,
- NoSuchMethodException, SecurityException {
+ final HostAndPort address, final String instanceName) throws
ClassNotFoundException,
+ InstantiationException, IllegalAccessException, IllegalArgumentException,
+ InvocationTargetException, NoSuchMethodException, SecurityException {
initializeMetrics(conf.getBoolean(Property.GENERAL_MICROMETER_ENABLED),
conf.getBoolean(Property.GENERAL_MICROMETER_JVM_METRICS_ENABLED),
-conf.get(Property.GENERAL_MICROMETER_FACTORY), appName, address);
+conf.get(Property.GENERAL_MICROMETER_FACTORY), appName, address,
instanceName);
}
private static void initializeMetrics(boolean enabled, boolean
jvmMetricsEnabled,
- String factoryClass, String appName, HostAndPort address) throws
ClassNotFoundException,
- InstantiationException, IllegalAccessException, IllegalArgumentException,
- InvocationTargetException, NoSuchMethodException, SecurityException {
+ String factoryClass, String appName, HostAndPort address, String
instanceName)
+ throws ClassNotFoundException, InstantiationException,
IllegalAccessException,
+ IllegalArgumentException, InvocationTargetException,
NoSuchMethodException,
+ SecurityException {
LOG.info("initializing metrics, enabled:{}, class:{}", enabled,
factoryClass);
@@ -77,6 +78,7 @@ public class MetricsUtil {
}
List tags = new ArrayList<>();
+ tags.add(Tag.of("instance.name", instanceName));
tags.add(Tag.of("process.name", processName));
if (address != null) {
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index 914f4fe112..88df289a56 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@ -268,7 +268,7 @@ public class CompactionCoordinator extends AbstractServer
try {
MetricsUtil.initializeMetrics(getContext().getConfiguration(),
this.applicationName,
- clientAddress);
+ clientAddress, getContext().getInstanceName());
} catch (ClassNotFoundException | InstantiationException |
IllegalAccessException
| IllegalArgumentException | InvocationTargetException |
NoSuchMethodException
| SecurityException e1) {
diff --git
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index d9cc4789d7..9662757d3d 100644
---
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@ -616,7 +616,7 @@ public class Compactor extends AbstractServer implements
MetricsProducer, Compac
try {
MetricsUtil.initializeMetrics(getContext().getConfiguration(),
this.applicationName,
- clientAddress);
+ clientAddress, getContext().getInstanceName());
}
