[accumulo] branch elasticity updated (9fa27a6d4d -> 9c7c467a56)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 9fa27a6d4d Modify merge and deleterows to use conditional mutations (#3875) add d021f3ca69 Adds property description formatting (#3877) add e6cbdb4418 Adds Static "named" constructors and updates GC (#3805) add c4c8c572c8 Merge branch '2.1' new 9c7c467a56 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/core/conf/Property.java| 246 ++--- .../org/apache/accumulo/core/gc/Reference.java | 7 + .../accumulo/core/gc/ReferenceDirectory.java | 2 +- .../org/apache/accumulo/core/gc/ReferenceFile.java | 32 ++- .../apache/accumulo/core/conf/PropertyTest.java| 4 + .../accumulo/server/gc/AllVolumesDirectory.java| 2 +- .../accumulo/server/metadata/ServerAmpleImpl.java | 4 +- .../accumulo/server/util/MetadataTableUtil.java| 2 +- .../main/java/org/apache/accumulo/gc/GCRun.java| 31 +-- .../accumulo/gc/GarbageCollectionAlgorithm.java| 16 +- .../apache/accumulo/gc/GarbageCollectionTest.java | 164 +- .../tableOps/bulkVer2/CleanUpBulkImport.java | 2 +- .../test/functional/GarbageCollectorIT.java| 2 +- 13 files changed, 293 insertions(+), 221 deletions(-)
[accumulo] 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 9c7c467a5614c095d4a78c3bd0e8440841e5669b
Merge: 9fa27a6d4d c4c8c572c8
Author: Daniel Roberts
AuthorDate: Wed Oct 25 03:45:44 2023 +
Merge branch 'main' into elasticity
.../org/apache/accumulo/core/conf/Property.java| 246 ++---
.../org/apache/accumulo/core/gc/Reference.java | 7 +
.../accumulo/core/gc/ReferenceDirectory.java | 2 +-
.../org/apache/accumulo/core/gc/ReferenceFile.java | 32 ++-
.../apache/accumulo/core/conf/PropertyTest.java| 4 +
.../accumulo/server/gc/AllVolumesDirectory.java| 2 +-
.../accumulo/server/metadata/ServerAmpleImpl.java | 4 +-
.../accumulo/server/util/MetadataTableUtil.java| 2 +-
.../main/java/org/apache/accumulo/gc/GCRun.java| 31 +--
.../accumulo/gc/GarbageCollectionAlgorithm.java| 16 +-
.../apache/accumulo/gc/GarbageCollectionTest.java | 164 +-
.../tableOps/bulkVer2/CleanUpBulkImport.java | 2 +-
.../test/functional/GarbageCollectorIT.java| 2 +-
13 files changed, 293 insertions(+), 221 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 0b4df706b0,58be89df05..d2dea79464
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -298,19 -297,16 +298,19 @@@ public enum Property
"3.0.0"),
GENERAL_MAX_SCANNER_RETRY_PERIOD("general.max.scanner.retry.period", "5s",
PropertyType.TIMEDURATION,
- "The maximum amount of time that a Scanner should wait before retrying
a failed RPC",
+ "The maximum amount of time that a Scanner should wait before retrying
a failed RPC.",
"1.7.3"),
+
GENERAL_MICROMETER_CACHE_METRICS_ENABLED("general.micrometer.cache.metrics.enabled",
"false",
- PropertyType.BOOLEAN, "Enables Caffeine Cache metrics functionality
using Micrometer",
++ PropertyType.BOOLEAN, "Enables Caffeine Cache metrics functionality
using Micrometer.",
+ "4.0.0"),
GENERAL_MICROMETER_ENABLED("general.micrometer.enabled", "false",
PropertyType.BOOLEAN,
- "Enables metrics functionality using Micrometer", "2.1.0"),
+ "Enables metrics functionality using Micrometer.", "2.1.0"),
GENERAL_MICROMETER_JVM_METRICS_ENABLED("general.micrometer.jvm.metrics.enabled",
"false",
- PropertyType.BOOLEAN, "Enables JVM metrics functionality using
Micrometer", "2.1.0"),
+ PropertyType.BOOLEAN, "Enables JVM metrics functionality using
Micrometer.", "2.1.0"),
GENERAL_MICROMETER_FACTORY("general.micrometer.factory", "",
PropertyType.CLASSNAME,
- "Name of class that implements MeterRegistryFactory", "2.1.0"),
+ "Name of class that implements MeterRegistryFactory.", "2.1.0"),
GENERAL_PROCESS_BIND_ADDRESS("general.process.bind.addr", "0.0.0.0",
PropertyType.STRING,
- "The local IP address to which this server should bind for sending and
receiving network traffic",
+ "The local IP address to which this server should bind for sending and
receiving network traffic.",
"3.0.0"),
// properties that are specific to manager server behavior
MANAGER_PREFIX("manager.", null, PropertyType.PREFIX,
@@@ -326,13 -322,8 +326,13 @@@
PropertyType.TIMEDURATION,
"Time to wait between scanning tablet states to identify tablets that
need to be assigned, un-assigned, migrated, etc.",
"2.1.2"),
+
MANAGER_TABLET_GROUP_WATCHER_SCAN_THREADS("manager.tablet.watcher.scan.threads.max",
"16",
+ PropertyType.COUNT,
+ "Maximum number of threads the TabletGroupWatcher will use in its
BatchScanner to"
- + " look for tablets that need maintenance",
++ + " look for tablets that need maintenance.",
+ "4.0.0"),
MANAGER_BULK_TIMEOUT("manager.bulk.timeout", "5m",
PropertyType.TIMEDURATION,
- "The time to wait for a tablet server to process a bulk import
request", "1.4.3"),
+ "The time to wait for a tablet server to process a bulk import
request.", "1.4.3"),
MANAGER_RENAME_THREADS("manager.rename.threadpool.size", "20",
PropertyType.COUNT,
"The number of threads to use when renaming user files during table
import or bulk ingest.",
"2.1.0"),
@@@ -383,13 -374,8 +383,13 @@@
"Maximum time manager will wait for tserver available threshold "
+ "to be reached before continuing. When set to 0 or less, will
block "
+ "indefinitely. Default is 0 to block indefinitely. Only valid
when tserver available "
- + "threshold is set greater than 0. Added with version 1.10",
+ + "threshold is set greater than 0.",
"1.10.0"),
+ MANAGER_SPLIT_WORKER_THREADS("manager.split.inspection.threadpool.size",
"8", PropertyType.COUNT,
+ "The
[accumulo] branch main updated: adds range to compactable file (#3883)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new b5b48c3e38 adds range to compactable file (#3883)
b5b48c3e38 is described below
commit b5b48c3e389186e1229be7b8e5ab9e30d87dd52d
Author: Keith Turner
AuthorDate: Tue Oct 24 20:50:52 2023 -0400
adds range to compactable file (#3883)
Now that tablet files have an associated range, this information should
be made available to compaction plugins. Making this information
available to plugins enable uses cases like selecting files with ranges
for compaction. This commit makes that information avialable by adding a
getRange method to CompactableFile
---
.../core/client/admin/compaction/CompactableFile.java | 19 +++
.../accumulo/core/metadata/CompactableFileImpl.java | 11 +++
2 files changed, 30 insertions(+)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactableFile.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactableFile.java
index baf5f04949..36b2d7b0ea 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactableFile.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactableFile.java
@@ -20,6 +20,7 @@ package org.apache.accumulo.core.client.admin.compaction;
import java.net.URI;
+import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.metadata.CompactableFileImpl;
/**
@@ -33,6 +34,16 @@ public interface CompactableFile {
public URI getUri();
+ /**
+ * @return A range associated with the file. If a file has an associated
range then Accumulo will
+ * limit reads to within the range. Not all files have an associated
range, it a file does
+ * not have a range then an infinite range is returned. The URI plus
this range uniquely
+ * identify a file.
+ *
+ * @since 3.1.0
+ */
+ public Range getRange();
+
public long getEstimatedSize();
public long getEstimatedEntries();
@@ -41,4 +52,12 @@ public interface CompactableFile {
return new CompactableFileImpl(uri, estimatedSize, estimatedEntries);
}
+ /**
+ * Creates a new CompactableFile object that implements this interface.
+ *
+ * @since 3.1.0
+ */
+ static CompactableFile create(URI uri, Range range, long estimatedSize, long
estimatedEntries) {
+return new CompactableFileImpl(uri, range, estimatedSize,
estimatedEntries);
+ }
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/CompactableFileImpl.java
b/core/src/main/java/org/apache/accumulo/core/metadata/CompactableFileImpl.java
index e3ff1a334a..b25e83dde3 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/CompactableFileImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/CompactableFileImpl.java
@@ -22,6 +22,7 @@ import java.net.URI;
import java.util.Objects;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
+import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
public class CompactableFileImpl implements CompactableFile {
@@ -34,6 +35,11 @@ public class CompactableFileImpl implements CompactableFile {
this.dataFileValue = new DataFileValue(size, entries);
}
+ public CompactableFileImpl(URI uri, Range range, long size, long entries) {
+this.storedTabletFile = StoredTabletFile.of(uri, range);
+this.dataFileValue = new DataFileValue(size, entries);
+ }
+
public CompactableFileImpl(StoredTabletFile storedTabletFile, DataFileValue
dataFileValue) {
this.storedTabletFile = Objects.requireNonNull(storedTabletFile);
this.dataFileValue = Objects.requireNonNull(dataFileValue);
@@ -44,6 +50,11 @@ public class CompactableFileImpl implements CompactableFile {
return storedTabletFile.getPath().toUri();
}
+ @Override
+ public Range getRange() {
+return storedTabletFile.getRange();
+ }
+
@Override
public String getFileName() {
return storedTabletFile.getFileName();
[accumulo] branch main updated: logs row ranges for files that have them (#3885)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 2028745941 logs row ranges for files that have them (#3885)
2028745941 is described below
commit 202874594185c7497960a43262b12d3ab9d78d7d
Author: Keith Turner
AuthorDate: Tue Oct 24 20:50:18 2023 -0400
logs row ranges for files that have them (#3885)
There was code that was logging only tablets file names. Now that files
can have a row range, need to log the range if it exists.
---
.../apache/accumulo/core/logging/TabletLogger.java | 16 +---
.../accumulo/core/metadata/AbstractTabletFile.java | 21 +
.../accumulo/core/metadata/CompactableFileImpl.java | 2 +-
.../apache/accumulo/core/metadata/TabletFile.java | 5 +
.../accumulo/tserver/tablet/CompactableImpl.java| 15 ---
.../org/apache/accumulo/test/ComprehensiveIT.java | 1 -
6 files changed, 44 insertions(+), 16 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
b/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
index 100cc318cc..e632971cf5 100644
--- a/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
+++ b/core/src/main/java/org/apache/accumulo/core/logging/TabletLogger.java
@@ -28,6 +28,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.client.admin.CompactionConfig;
import org.apache.accumulo.core.client.admin.compaction.CompactableFile;
import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.CompactableFileImpl;
import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.TabletFile;
@@ -117,32 +118,33 @@ public class TabletLogger {
* Lazily converts TableFile to file names. The lazy part is really
important because when it is
* not called with log.isDebugEnabled().
*/
- private static Collection asFileNames(Collection
files) {
-return Collections2.transform(files, CompactableFile::getFileName);
+ private static Collection
asMinimalString(Collection files) {
+return Collections2.transform(files,
+cf -> CompactableFileImpl.toStoredTabletFile(cf).toMinimalString());
}
public static void selected(KeyExtent extent, CompactionKind kind,
Collection inputs) {
fileLog.trace("{} changed compaction selection set for {} new set {}",
extent, kind,
-Collections2.transform(inputs, StoredTabletFile::getFileName));
+Collections2.transform(inputs, StoredTabletFile::toMinimalString));
}
public static void compacting(KeyExtent extent, CompactionJob job,
CompactionConfig config) {
if (fileLog.isDebugEnabled()) {
if (config == null) {
fileLog.debug("Compacting {} on {} for {} from {} size {}", extent,
job.getExecutor(),
-job.getKind(), asFileNames(job.getFiles()),
getSize(job.getFiles()));
+job.getKind(), asMinimalString(job.getFiles()),
getSize(job.getFiles()));
} else {
fileLog.debug("Compacting {} on {} for {} from {} size {} config {}",
extent,
-job.getExecutor(), job.getKind(), asFileNames(job.getFiles()),
getSize(job.getFiles()),
-config);
+job.getExecutor(), job.getKind(), asMinimalString(job.getFiles()),
+getSize(job.getFiles()), config);
}
}
}
public static void compacted(KeyExtent extent, CompactionJob job,
StoredTabletFile output) {
fileLog.debug("Compacted {} for {} created {} from {}", extent,
job.getKind(), output,
-asFileNames(job.getFiles()));
+asMinimalString(job.getFiles()));
}
public static void flushed(KeyExtent extent, Optional
newDatafile) {
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/AbstractTabletFile.java
b/core/src/main/java/org/apache/accumulo/core/metadata/AbstractTabletFile.java
index 0957c07a9d..ae0d46fbf2 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/AbstractTabletFile.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/AbstractTabletFile.java
@@ -20,6 +20,7 @@ package org.apache.accumulo.core.metadata;
import java.util.Objects;
+import org.apache.accumulo.core.data.ByteSequence;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.hadoop.fs.Path;
@@ -82,4 +83,24 @@ public abstract class AbstractTabletFile>
return row.length() > 0 && row.byteAt(row.length() - 1) == (byte) 0x00;
}
+ private static String stripZeroTail(ByteSequence row) {
+if (row.byteAt(row.length() - 1) == (byte) 0x00) {
+ return row.subSequence(0, row.length() - 1).toString();
+}
+return row.toString();
+ }
+
+
[accumulo] branch elasticity updated: Modify merge and deleterows to use conditional mutations (#3875)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 9fa27a6d4d Modify merge and deleterows to use conditional mutations
(#3875)
9fa27a6d4d is described below
commit 9fa27a6d4d8542a48c2a74849b169269e0a4609c
Author: Keith Turner
AuthorDate: Tue Oct 24 20:49:29 2023 -0400
Modify merge and deleterows to use conditional mutations (#3875)
Added a new FATE repo that acquires operations ids on all of the tablets
in the merge range. This causes the tablet group watcher to unload
tablets. This new repo waits for all tablets to not have a location.
Modified the existing merge and delete rows FATE repos to use ample and
conditional mutations. Using ample simplified the code.
Modified the delete rows FATE repo to only delete or fence files, but
not merge tablets. Had the deleterows code call the merge code to
delete tablets.
Removed all merge code from tablet group watcher and related classes
that was related to unassigning merging tablets. Setting the operation
ids takes care of this now.
Modified the final FATE repo to delete operation ids.
MergeIT and DeleteRowsIT are passing with this change
---
.../core/manager/state/TabletManagement.java | 2 +-
.../accumulo/core/metadata/schema/Ample.java | 43 +-
.../metadata/schema/TabletMetadataBuilder.java | 6 +
.../core/metadata/schema/TabletMutatorBase.java| 21 +
.../server/manager/state/CurrentState.java | 3 -
.../accumulo/server/manager/state/MergeInfo.java | 107 -
.../accumulo/server/manager/state/MergeState.java | 41 --
.../manager/state/TabletManagementIterator.java| 47 --
.../metadata/ConditionalTabletsMutatorImpl.java| 7 +-
.../server/manager/state/MergeInfoTest.java| 206 -
.../accumulo/manager/FateServiceHandler.java | 2 +-
.../java/org/apache/accumulo/manager/Manager.java | 96 +---
.../accumulo/manager/TabletGroupWatcher.java | 14 +-
.../manager/tableOps/merge/DeleteRows.java | 491 +
.../manager/tableOps/merge/FinishTableRangeOp.java | 85 ++--
.../accumulo/manager/tableOps/merge/MergeInfo.java | 127 ++
.../manager/tableOps/merge/MergeTablets.java | 371 +++-
.../manager/tableOps/merge/ReserveTablets.java | 124 ++
.../manager/tableOps/merge/TableRangeOp.java | 61 +--
.../manager/tableOps/merge/WaitForOffline.java | 81
.../org/apache/accumulo/test/ComprehensiveIT.java | 1 -
.../accumulo/test/functional/DeleteRowsIT.java | 1 +
.../functional/TabletManagementIteratorIT.java | 19 -
23 files changed, 665 insertions(+), 1291 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/manager/state/TabletManagement.java
b/core/src/main/java/org/apache/accumulo/core/manager/state/TabletManagement.java
index 647f6f9412..495f3e2ef0 100644
---
a/core/src/main/java/org/apache/accumulo/core/manager/state/TabletManagement.java
+++
b/core/src/main/java/org/apache/accumulo/core/manager/state/TabletManagement.java
@@ -53,7 +53,7 @@ public class TabletManagement {
private static final Text EMPTY = new Text("");
public static enum ManagementAction {
-BAD_STATE, NEEDS_COMPACTING, NEEDS_LOCATION_UPDATE, IS_MERGING,
NEEDS_SPLITTING;
+BAD_STATE, NEEDS_COMPACTING, NEEDS_LOCATION_UPDATE, NEEDS_SPLITTING;
}
public static void addActions(final SortedMap decodedRow,
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
index 793e5b5d9b..a23e6da08e 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
@@ -22,12 +22,14 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import java.util.UUID;
import java.util.function.Predicate;
import java.util.stream.Stream;
import org.apache.accumulo.core.client.ConditionalWriter;
import org.apache.accumulo.core.client.admin.TabletHostingGoal;
+import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
@@ -375,6 +377,14 @@ public interface Ample {
T putSelectedFiles(SelectedFiles selectedFiles);
T deleteSelectedFiles();
+
+/**
+ * Deletes all the columns in the keys.
+ *
+ * @throws IllegalArgumentException if rows in keys do not match tablet
row or column visibility
+ * is not empty
+ */
+T deleteAll(Set keys);
}
interface TabletMutator
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit c4c8c572c8be14d882899f8f9370700227e87423
Merge: c14a97fe8c e6cbdb4418
Author: Daniel Roberts
AuthorDate: Tue Oct 24 17:21:21 2023 +
Merge branch '2.1'
.../org/apache/accumulo/core/conf/Property.java| 243 ++---
.../org/apache/accumulo/core/gc/Reference.java | 7 +
.../accumulo/core/gc/ReferenceDirectory.java | 2 +-
.../org/apache/accumulo/core/gc/ReferenceFile.java | 32 ++-
.../apache/accumulo/core/conf/PropertyTest.java| 4 +
.../accumulo/server/gc/AllVolumesDirectory.java| 2 +-
.../accumulo/server/metadata/ServerAmpleImpl.java | 4 +-
.../accumulo/server/util/ManagerMetadataUtil.java | 4 +
.../accumulo/server/util/MetadataTableUtil.java| 2 +-
.../main/java/org/apache/accumulo/gc/GCRun.java| 31 +--
.../accumulo/gc/GarbageCollectionAlgorithm.java| 16 +-
.../apache/accumulo/gc/GarbageCollectionTest.java | 164 --
.../accumulo/manager/TabletGroupWatcher.java | 2 +-
.../tableOps/bulkVer2/CleanUpBulkImport.java | 2 +-
.../test/functional/GarbageCollectorIT.java| 2 +-
15 files changed, 296 insertions(+), 221 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 903c5d7443,bafc826c5a..58be89df05
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -60,28 -61,28 +60,28 @@@ public enum Property
@Sensitive
RPC_SSL_KEYSTORE_PASSWORD("rpc.javax.net.ssl.keyStorePassword", "",
PropertyType.STRING,
"Password used to encrypt the SSL private keystore. "
- + "Leave blank to use the Accumulo instance secret",
+ + "Leave blank to use the Accumulo instance secret.",
"1.6.0"),
RPC_SSL_KEYSTORE_TYPE("rpc.javax.net.ssl.keyStoreType", "jks",
PropertyType.STRING,
- "Type of SSL keystore", "1.6.0"),
+ "Type of SSL keystore.", "1.6.0"),
RPC_SSL_TRUSTSTORE_PATH("rpc.javax.net.ssl.trustStore", "",
PropertyType.PATH,
- "Path of the truststore file for the root cert", "1.6.0"),
+ "Path of the truststore file for the root cert.", "1.6.0"),
@Sensitive
RPC_SSL_TRUSTSTORE_PASSWORD("rpc.javax.net.ssl.trustStorePassword", "",
PropertyType.STRING,
- "Password used to encrypt the SSL truststore. Leave blank to use no
password", "1.6.0"),
+ "Password used to encrypt the SSL truststore. Leave blank to use no
password.", "1.6.0"),
RPC_SSL_TRUSTSTORE_TYPE("rpc.javax.net.ssl.trustStoreType", "jks",
PropertyType.STRING,
- "Type of SSL truststore", "1.6.0"),
+ "Type of SSL truststore.", "1.6.0"),
RPC_USE_JSSE("rpc.useJsse", "false", PropertyType.BOOLEAN,
"Use JSSE system properties to configure SSL rather than the " +
RPC_PREFIX.getKey()
- + "javax.net.ssl.* Accumulo properties",
+ + "javax.net.ssl.* Accumulo properties.",
"1.6.0"),
RPC_SSL_CIPHER_SUITES("rpc.ssl.cipher.suites", "", PropertyType.STRING,
- "Comma separated list of cipher suites that can be used by accepted
connections", "1.6.1"),
+ "Comma separated list of cipher suites that can be used by accepted
connections.", "1.6.1"),
- RPC_SSL_ENABLED_PROTOCOLS("rpc.ssl.server.enabled.protocols", "TLSv1.2",
PropertyType.STRING,
+ RPC_SSL_ENABLED_PROTOCOLS("rpc.ssl.server.enabled.protocols", "TLSv1.3",
PropertyType.STRING,
- "Comma separated list of protocols that can be used to accept
connections", "1.6.2"),
+ "Comma separated list of protocols that can be used to accept
connections.", "1.6.2"),
- RPC_SSL_CLIENT_PROTOCOL("rpc.ssl.client.protocol", "TLSv1.2",
PropertyType.STRING,
+ RPC_SSL_CLIENT_PROTOCOL("rpc.ssl.client.protocol", "TLSv1.3",
PropertyType.STRING,
- "The protocol used to connect to a secure server, must be in the list
of enabled protocols "
- + "on the server side (rpc.ssl.server.enabled.protocols)",
+ "The protocol used to connect to a secure server. Must be in the list
of enabled protocols "
+ + "on the server side `rpc.ssl.server.enabled.protocols`.",
"1.6.2"),
RPC_SASL_QOP("rpc.sasl.qop", "auth", PropertyType.STRING,
"The quality of protection to be used with SASL. Valid values are
'auth', 'auth-int',"
@@@ -250,7 -265,12 +250,7 @@@
"Enables tracing functionality using OpenTelemetry (assuming
OpenTelemetry is configured).",
"2.1.0"),
GENERAL_THREADPOOL_SIZE("general.server.threadpool.size", "1",
PropertyType.COUNT,
- "The number of threads to use for server-internal scheduled tasks",
"2.1.0"),
+ "The number of threads to use for server-internal scheduled tasks.",
"2.1.0"),
- @Deprecated(since = "2.1.0")
- @ReplacedBy(property = GENERAL_THREADPOOL_SIZE)
[accumulo] branch main updated (c14a97fe8c -> c4c8c572c8)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from c14a97fe8c Deprecate rate limiting properties (#3882) add d021f3ca69 Adds property description formatting (#3877) add e6cbdb4418 Adds Static "named" constructors and updates GC (#3805) new c4c8c572c8 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/core/conf/Property.java| 243 ++--- .../org/apache/accumulo/core/gc/Reference.java | 7 + .../accumulo/core/gc/ReferenceDirectory.java | 2 +- .../org/apache/accumulo/core/gc/ReferenceFile.java | 32 ++- .../apache/accumulo/core/conf/PropertyTest.java| 4 + .../accumulo/server/gc/AllVolumesDirectory.java| 2 +- .../accumulo/server/metadata/ServerAmpleImpl.java | 4 +- .../accumulo/server/util/ManagerMetadataUtil.java | 4 + .../accumulo/server/util/MetadataTableUtil.java| 2 +- .../main/java/org/apache/accumulo/gc/GCRun.java| 31 +-- .../accumulo/gc/GarbageCollectionAlgorithm.java| 16 +- .../apache/accumulo/gc/GarbageCollectionTest.java | 164 -- .../accumulo/manager/TabletGroupWatcher.java | 2 +- .../tableOps/bulkVer2/CleanUpBulkImport.java | 2 +- .../test/functional/GarbageCollectorIT.java| 2 +- 15 files changed, 296 insertions(+), 221 deletions(-)
[accumulo] branch elasticity updated: Enable reporting of TabletManagementIterator errors (#3872)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 07313e3b41 Enable reporting of TabletManagementIterator errors (#3872)
07313e3b41 is described below
commit 07313e3b41539aab327db79cd3348dc3a00684c4
Author: Dave Marion
AuthorDate: Tue Oct 24 16:00:51 2023 -0400
Enable reporting of TabletManagementIterator errors (#3872)
Enable the TabletManagementIterator to report error's back
to the TabletGroupWatcher when trying to determine actions
that should be taken for an extent. Increment metric gauges
when an error occurs.
Fixes #3469
---
.../core/manager/state/TabletManagement.java | 39 +++---
.../accumulo/core/metrics/MetricsProducer.java | 26 +++
.../manager/state/TabletManagementIterator.java| 36 +---
.../manager/state/TabletManagementScanner.java | 4 +--
.../server/manager/state/ZooTabletStateStore.java | 25 --
.../server/manager/state/TabletManagementTest.java | 20 +++
.../gc/GarbageCollectWriteAheadLogsTest.java | 6 ++--
.../java/org/apache/accumulo/manager/Manager.java | 8 ++---
.../accumulo/manager/TabletGroupWatcher.java | 19 +--
.../accumulo/manager/metrics/ManagerMetrics.java | 30 +
.../apache/accumulo/test/metrics/MetricsIT.java| 4 ++-
11 files changed, 179 insertions(+), 38 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/manager/state/TabletManagement.java
b/core/src/main/java/org/apache/accumulo/core/manager/state/TabletManagement.java
index a687de873f..647f6f9412 100644
---
a/core/src/main/java/org/apache/accumulo/core/manager/state/TabletManagement.java
+++
b/core/src/main/java/org/apache/accumulo/core/manager/state/TabletManagement.java
@@ -47,6 +47,7 @@ public class TabletManagement {
ColumnType.HOSTING_GOAL, ColumnType.HOSTING_REQUESTED,
ColumnType.FILES, ColumnType.LAST,
ColumnType.OPID, ColumnType.ECOMP, ColumnType.DIR,
ColumnType.SELECTED);
+ private static final Text ERROR_COLUMN_NAME = new Text("ERROR");
private static final Text REASONS_COLUMN_NAME = new Text("REASONS");
private static final Text EMPTY = new Text("");
@@ -62,12 +63,20 @@ public class TabletManagement {
decodedRow.put(reasonsKey, reasonsValue);
}
- public final Set actions;
- public final TabletMetadata tabletMetadata;
+ public static void addError(final SortedMap decodedRow, final
Exception error) {
+final Key errorKey = new Key(decodedRow.firstKey().getRow(),
ERROR_COLUMN_NAME, EMPTY);
+final Value errorValue = new Value(error.getMessage());
+decodedRow.put(errorKey, errorValue);
+ }
+
+ private final Set actions;
+ private final TabletMetadata tabletMetadata;
+ private final String errorMessage;
- public TabletManagement(Set actions, TabletMetadata tm) {
+ public TabletManagement(Set actions, TabletMetadata tm,
String errorMessage) {
this.actions = actions;
this.tabletMetadata = tm;
+this.errorMessage = errorMessage;
}
public TabletManagement(Key wholeRowKey, Value wholeRowValue) throws
IOException {
@@ -77,9 +86,21 @@ public class TabletManagement {
public TabletManagement(Key wholeRowKey, Value wholeRowValue, boolean
saveKV) throws IOException {
final SortedMap decodedRow =
WholeRowIterator.decodeRow(wholeRowKey, wholeRowValue);
Text row = decodedRow.firstKey().getRow();
-Value val = decodedRow.remove(new Key(row, REASONS_COLUMN_NAME, EMPTY));
+// Decode any errors that happened on the TabletServer
+Value errorValue = decodedRow.remove(new Key(row, ERROR_COLUMN_NAME,
EMPTY));
+if (errorValue != null) {
+ this.errorMessage = errorValue.toString();
+} else {
+ this.errorMessage = null;
+}
+// Decode the ManagementActions if it exists
+Value actionValue = decodedRow.remove(new Key(row, REASONS_COLUMN_NAME,
EMPTY));
Set actions = new HashSet<>();
-Splitter.on(',').split(val.toString()).forEach(a ->
actions.add(ManagementAction.valueOf(a)));
+if (actionValue != null) {
+ Splitter.on(',').split(actionValue.toString())
+ .forEach(a -> actions.add(ManagementAction.valueOf(a)));
+}
+
TabletMetadata tm =
TabletMetadata.convertRow(decodedRow.entrySet().iterator(),
CONFIGURED_COLUMNS, saveKV, true);
this.actions = actions;
@@ -94,6 +115,14 @@ public class TabletManagement {
return tabletMetadata;
}
+ /**
+ * @return exception message if an exception was thrown while computing this
tablets management
+ * actions OR null if no exception was seen
+ */
+ public String getErrorMessage() {
+return errorMessage;
+ }
+
@Override
public String toString() {
return actions.toString()
[accumulo] branch elasticity updated: Fix issue in TGW when shutting down all tservers (#3866)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new d2a9ef609b Fix issue in TGW when shutting down all tservers (#3866)
d2a9ef609b is described below
commit d2a9ef609bcb30f54a4afda5968a5101a9f0
Author: Dave Marion
AuthorDate: Tue Oct 24 14:18:01 2023 -0400
Fix issue in TGW when shutting down all tservers (#3866)
Fixes an issue where the root table was being
unloaded due to an EventCoordinator initiatied
run of the TGW while the metadata table was still
hosted.
Fixes #3861
---
.../accumulo/manager/TabletGroupWatcher.java | 15 +++--
.../test/functional/ManagerAssignmentIT.java | 38 --
.../accumulo/test/functional/ShutdownIT.java | 2 --
.../test/functional/SslWithClientAuthIT.java | 2 --
4 files changed, 27 insertions(+), 30 deletions(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index 29fe40cdce..da6d4a4f9e 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@ -298,6 +298,15 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
IOException {
TableMgmtStats tableMgmtStats = new TableMgmtStats();
+final boolean shuttingDownAllTabletServers =
+manager.serversToShutdown.equals(currentTServers.keySet());
+if (shuttingDownAllTabletServers && !isFullScan) {
+ // If we are shutting down all of the TabletServers, then don't process
any events
+ // from the EventCoordinator.
+ LOG.debug("Partial scan requested, but aborted due to shutdown of all
TabletServers");
+ return tableMgmtStats;
+}
+
int unloaded = 0;
Map currentMerges = new HashMap<>();
@@ -393,8 +402,8 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
}
// if we are shutting down all the tabletservers, we have to do it in
order
- if (isFullScan && (goal == TabletGoalState.SUSPENDED && state ==
TabletState.HOSTED)
- && manager.serversToShutdown.equals(currentTServers.keySet())) {
+ if (shuttingDownAllTabletServers
+ && (goal == TabletGoalState.SUSPENDED && state ==
TabletState.HOSTED)) {
if (dependentWatcher != null) {
// If the dependentWatcher is for the user tables, check to see
// that user tables exist.
@@ -505,6 +514,8 @@ abstract class TabletGroupWatcher extends
AccumuloDaemonThread {
TServerConnection client =
manager.tserverSet.getConnection(location.getServerInstance());
if (client != null) {
+LOG.debug("Requesting tserver {} unload tablet {}",
location.getServerInstance(),
+tm.getExtent());
client.unloadTablet(manager.managerLock, tm.getExtent(),
goal.howUnload(),
manager.getSteadyTime());
tableMgmtStats.totalUnloaded++;
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java
index 5e60e7ca2c..0f82311d7a 100644
---
a/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java
@@ -26,7 +26,6 @@ import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
-import java.time.Duration;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -85,20 +84,13 @@ import org.apache.accumulo.test.util.Wait;
import org.apache.hadoop.io.Text;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import com.google.common.collect.Iterables;
import com.google.common.net.HostAndPort;
-@Disabled // ELASTICITY_TODO
public class ManagerAssignmentIT extends SharedMiniClusterBase {
- @Override
- protected Duration defaultTimeout() {
-return Duration.ofMinutes(2);
- }
-
@BeforeAll
public static void beforeAll() throws Exception {
SharedMiniClusterBase.startMiniClusterWithConfig((cfg, core) -> {
@@ -114,13 +106,8 @@ public class ManagerAssignmentIT extends
SharedMiniClusterBase {
@BeforeEach
public void before() throws Exception {
try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
- if (client.instanceOperations().getTabletServers().size() ==
[accumulo] branch 2.1 updated: Adds Static "named" constructors and updates GC (#3805)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new e6cbdb4418 Adds Static "named" constructors and updates GC (#3805)
e6cbdb4418 is described below
commit e6cbdb4418d50e6c3c99e166a0390e483401fded
Author: Daniel Roberts
AuthorDate: Tue Oct 24 12:16:31 2023 -0400
Adds Static "named" constructors and updates GC (#3805)
* Adds static named constructors to ensure that inuse scan candidates are
not removed.
* Fix possible race condition with InUse Candidates
This change writes the gcCandidates twice when performing a major
compaction to ensure that valid candidates were not removed before the
tablet mutation had completed.
Fixes: #3802
* Refactored test method name
Renamed `assertRemoved` to `assertFileRemoved` to convey that the
candidate is now an hdfs file reference that has been deleted by the GC
---
.../org/apache/accumulo/core/gc/Reference.java | 7 +
.../accumulo/core/gc/ReferenceDirectory.java | 2 +-
.../org/apache/accumulo/core/gc/ReferenceFile.java | 17 +-
.../accumulo/server/gc/AllVolumesDirectory.java| 2 +-
.../accumulo/server/metadata/ServerAmpleImpl.java | 4 +-
.../accumulo/server/util/ManagerMetadataUtil.java | 4 +
.../accumulo/server/util/MetadataTableUtil.java| 4 +-
.../main/java/org/apache/accumulo/gc/GCRun.java| 32 ++--
.../accumulo/gc/GarbageCollectionAlgorithm.java| 16 +-
.../apache/accumulo/gc/GarbageCollectionTest.java | 178 +
.../accumulo/manager/TabletGroupWatcher.java | 2 +-
.../tableOps/bulkVer1/CleanUpBulkImport.java | 2 +-
.../tableOps/bulkVer2/CleanUpBulkImport.java | 2 +-
.../accumulo/manager/upgrade/Upgrader9to10.java| 4 +-
.../manager/upgrade/Upgrader9to10Test.java | 9 +-
.../test/functional/GarbageCollectorIT.java| 2 +-
16 files changed, 181 insertions(+), 106 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/gc/Reference.java
b/core/src/main/java/org/apache/accumulo/core/gc/Reference.java
index cdffbd7eeb..4c67bfd31d 100644
--- a/core/src/main/java/org/apache/accumulo/core/gc/Reference.java
+++ b/core/src/main/java/org/apache/accumulo/core/gc/Reference.java
@@ -31,6 +31,11 @@ public interface Reference {
*/
boolean isDirectory();
+ /**
+ * Only return true if the reference is a scan.
+ */
+ boolean isScan();
+
/**
* Get the {@link TableId} of the reference.
*/
@@ -42,6 +47,8 @@ public interface Reference {
* {@link
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily}
* A directory will be read from the "srv:dir" column family:
* {@link
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily}
+ * A scan will be read from the Tablet "scan" column family:
+ * {@link
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ScanFileColumnFamily}
*/
String getMetadataEntry();
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/gc/ReferenceDirectory.java
b/core/src/main/java/org/apache/accumulo/core/gc/ReferenceDirectory.java
index b9a6589d9f..5491020aa4 100644
--- a/core/src/main/java/org/apache/accumulo/core/gc/ReferenceDirectory.java
+++ b/core/src/main/java/org/apache/accumulo/core/gc/ReferenceDirectory.java
@@ -28,7 +28,7 @@ public class ReferenceDirectory extends ReferenceFile {
private final String tabletDir; // t-0003
public ReferenceDirectory(TableId tableId, String dirName) {
-super(tableId, dirName);
+super(tableId, dirName, false);
MetadataSchema.TabletsSection.ServerColumnFamily.validateDirCol(dirName);
this.tabletDir = dirName;
}
diff --git a/core/src/main/java/org/apache/accumulo/core/gc/ReferenceFile.java
b/core/src/main/java/org/apache/accumulo/core/gc/ReferenceFile.java
index 7f796e8de9..b9eece90d5 100644
--- a/core/src/main/java/org/apache/accumulo/core/gc/ReferenceFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/gc/ReferenceFile.java
@@ -29,13 +29,23 @@ import org.apache.accumulo.core.data.TableId;
public class ReferenceFile implements Reference, Comparable {
// parts of an absolute URI, like "hdfs://1.2.3.4/accumulo/tables/2a/t-0003"
public final TableId tableId; // 2a
+ public final boolean isScan;
// the exact string that is stored in the metadata
protected final String metadataEntry;
- public ReferenceFile(TableId tableId, String metadataEntry) {
+ protected ReferenceFile(TableId tableId, String metadataEntry, boolean
isScan) {
this.tableId = Objects.requireNonNull(tableId);
this.metadataEntry = Objects.requireNonNull(metadataEntry);
+this.isScan = isScan;
+ }
+
+ public static ReferenceFile forFile(TableId tableId, String
[accumulo] branch 2.1 updated: Adds property description formatting (#3877)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new d021f3ca69 Adds property description formatting (#3877)
d021f3ca69 is described below
commit d021f3ca69c290a562494f19aebb66315a61abd8
Author: Daniel Roberts
AuthorDate: Tue Oct 24 12:12:41 2023 -0400
Adds property description formatting (#3877)
* Adds property description formatting
Adds a test to ensure that property descriptions end with a period.
---
.../org/apache/accumulo/core/conf/Property.java| 326 ++---
.../apache/accumulo/core/conf/PropertyTest.java| 4 +
2 files changed, 167 insertions(+), 163 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 191f411396..bafc826c5a 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -49,7 +49,7 @@ public enum Property {
// across all nodes in an instance)
RPC_PREFIX("rpc.", null, PropertyType.PREFIX,
"Properties in this category related to the configuration of SSL keys
for"
- + " RPC. See also instance.ssl.enabled",
+ + " RPC. See also `instance.ssl.enabled`.",
"1.6.0"),
RPC_BACKLOG("rpc.backlog", "50", PropertyType.COUNT,
"Configures the TCP backlog for the server side sockets created by
Thrift."
@@ -57,36 +57,36 @@ public enum Property {
+ " will use the Thrift default value.",
"2.1.3"),
RPC_SSL_KEYSTORE_PATH("rpc.javax.net.ssl.keyStore", "", PropertyType.PATH,
- "Path of the keystore file for the server's private SSL key", "1.6.0"),
+ "Path of the keystore file for the server's private SSL key.", "1.6.0"),
@Sensitive
RPC_SSL_KEYSTORE_PASSWORD("rpc.javax.net.ssl.keyStorePassword", "",
PropertyType.STRING,
"Password used to encrypt the SSL private keystore. "
- + "Leave blank to use the Accumulo instance secret",
+ + "Leave blank to use the Accumulo instance secret.",
"1.6.0"),
RPC_SSL_KEYSTORE_TYPE("rpc.javax.net.ssl.keyStoreType", "jks",
PropertyType.STRING,
- "Type of SSL keystore", "1.6.0"),
+ "Type of SSL keystore.", "1.6.0"),
RPC_SSL_TRUSTSTORE_PATH("rpc.javax.net.ssl.trustStore", "",
PropertyType.PATH,
- "Path of the truststore file for the root cert", "1.6.0"),
+ "Path of the truststore file for the root cert.", "1.6.0"),
@Sensitive
RPC_SSL_TRUSTSTORE_PASSWORD("rpc.javax.net.ssl.trustStorePassword", "",
PropertyType.STRING,
- "Password used to encrypt the SSL truststore. Leave blank to use no
password", "1.6.0"),
+ "Password used to encrypt the SSL truststore. Leave blank to use no
password.", "1.6.0"),
RPC_SSL_TRUSTSTORE_TYPE("rpc.javax.net.ssl.trustStoreType", "jks",
PropertyType.STRING,
- "Type of SSL truststore", "1.6.0"),
+ "Type of SSL truststore.", "1.6.0"),
RPC_USE_JSSE("rpc.useJsse", "false", PropertyType.BOOLEAN,
"Use JSSE system properties to configure SSL rather than the " +
RPC_PREFIX.getKey()
- + "javax.net.ssl.* Accumulo properties",
+ + "javax.net.ssl.* Accumulo properties.",
"1.6.0"),
RPC_SSL_CIPHER_SUITES("rpc.ssl.cipher.suites", "", PropertyType.STRING,
- "Comma separated list of cipher suites that can be used by accepted
connections", "1.6.1"),
+ "Comma separated list of cipher suites that can be used by accepted
connections.", "1.6.1"),
RPC_SSL_ENABLED_PROTOCOLS("rpc.ssl.server.enabled.protocols", "TLSv1.2",
PropertyType.STRING,
- "Comma separated list of protocols that can be used to accept
connections", "1.6.2"),
+ "Comma separated list of protocols that can be used to accept
connections.", "1.6.2"),
RPC_SSL_CLIENT_PROTOCOL("rpc.ssl.client.protocol", "TLSv1.2",
PropertyType.STRING,
- "The protocol used to connect to a secure server, must be in the list of
enabled protocols "
- + "on the server side (rpc.ssl.server.enabled.protocols)",
+ "The protocol used to connect to a secure server. Must be in the list of
enabled protocols "
+ + "on the server side `rpc.ssl.server.enabled.protocols`.",
"1.6.2"),
RPC_SASL_QOP("rpc.sasl.qop", "auth", PropertyType.STRING,
"The quality of protection to be used with SASL. Valid values are
'auth', 'auth-int',"
- + " and 'auth-conf'",
+ + " and 'auth-conf'.",
"1.7.0"),
// instance properties (must be the same for every node in an instance)
@@ -95,11 +95,11 @@ public enum Property {
+ "This is enforced and servers won't be able to communicate if
these differ.",
"1.3.5"),
INSTANCE_ZK_HOST("instance.zookeeper.host", "localhost:2181",
PropertyType.HOSTLIST,
-
[accumulo] branch elasticity updated: Add @Disabled annotation to failing tests (#3884)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new e39bdb6eac Add @Disabled annotation to failing tests (#3884)
e39bdb6eac is described below
commit e39bdb6eac40a3465ad49241127959b271894b3b
Author: Dave Marion
AuthorDate: Tue Oct 24 11:56:51 2023 -0400
Add @Disabled annotation to failing tests (#3884)
Fixes #3834
---
test/src/main/java/org/apache/accumulo/test/AuditMessageIT.java | 2 ++
test/src/main/java/org/apache/accumulo/test/CleanWalIT.java | 2 ++
test/src/main/java/org/apache/accumulo/test/GarbageCollectWALIT.java | 2 ++
test/src/main/java/org/apache/accumulo/test/LargeSplitRowIT.java | 2 ++
test/src/main/java/org/apache/accumulo/test/LocatorIT.java | 2 ++
.../org/apache/accumulo/test/ManagerRepairsDualAssignmentIT.java | 2 ++
test/src/main/java/org/apache/accumulo/test/MetaRecoveryIT.java | 2 ++
.../src/main/java/org/apache/accumulo/test/MultiTableRecoveryIT.java | 2 ++
.../org/apache/accumulo/test/RecoveryCompactionsAreFlushesIT.java| 2 ++
test/src/main/java/org/apache/accumulo/test/ScanConsistencyIT.java | 2 ++
test/src/main/java/org/apache/accumulo/test/SplitRecoveryIT.java | 2 ++
test/src/main/java/org/apache/accumulo/test/UnusedWALIT.java | 2 ++
.../main/java/org/apache/accumulo/test/VerifySerialRecoveryIT.java | 2 ++
test/src/main/java/org/apache/accumulo/test/VolumeIT.java| 2 ++
test/src/main/java/org/apache/accumulo/test/WaitForBalanceIT.java| 2 ++
test/src/main/java/org/apache/accumulo/test/WriteAfterCloseIT.java | 2 ++
.../apache/accumulo/test/compaction/CompactionConfigChangeIT.java| 2 ++
.../org/apache/accumulo/test/compaction/CompactionExecutorIT.java| 2 ++
.../apache/accumulo/test/compaction/ExternalCompactionMetricsIT.java | 2 ++
.../java/org/apache/accumulo/test/functional/AccumuloClientIT.java | 2 ++
.../org/apache/accumulo/test/functional/AssignLocationModeIT.java| 2 ++
.../java/org/apache/accumulo/test/functional/BigRootTabletIT.java| 2 ++
.../main/java/org/apache/accumulo/test/functional/BulkFailureIT.java | 2 ++
.../main/java/org/apache/accumulo/test/functional/CleanTmpIT.java| 2 ++
.../java/org/apache/accumulo/test/functional/DeleteRowsSplitIT.java | 2 ++
.../main/java/org/apache/accumulo/test/functional/DurabilityIT.java | 2 ++
.../java/org/apache/accumulo/test/functional/FateConcurrencyIT.java | 2 ++
.../java/org/apache/accumulo/test/functional/FileMetadataIT.java | 2 ++
.../main/java/org/apache/accumulo/test/functional/FlushNoFileIT.java | 2 ++
.../java/org/apache/accumulo/test/functional/GarbageCollectorIT.java | 2 ++
.../accumulo/test/functional/GarbageCollectorTrashDefaultIT.java | 2 ++
.../accumulo/test/functional/GarbageCollectorTrashEnabledIT.java | 2 ++
.../functional/GarbageCollectorTrashEnabledWithCustomPolicyIT.java | 2 ++
.../org/apache/accumulo/test/functional/ManagerAssignmentIT.java | 2 ++
.../org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java | 2 ++
.../java/org/apache/accumulo/test/functional/MetadataMaxFilesIT.java | 2 ++
.../org/apache/accumulo/test/functional/RegexGroupBalanceIT.java | 2 ++
.../src/main/java/org/apache/accumulo/test/functional/RestartIT.java | 5 +
.../org/apache/accumulo/test/functional/SessionDurabilityIT.java | 2 ++
.../main/java/org/apache/accumulo/test/functional/ShutdownIT.java| 2 ++
.../java/org/apache/accumulo/test/functional/SplitRecoveryIT.java| 2 ++
.../org/apache/accumulo/test/functional/SslWithClientAuthIT.java | 2 ++
.../src/main/java/org/apache/accumulo/test/functional/SummaryIT.java | 2 ++
.../apache/accumulo/test/functional/WriteAheadLogEncryptedIT.java| 2 ++
.../java/org/apache/accumulo/test/functional/WriteAheadLogIT.java| 2 ++
.../java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java| 2 ++
46 files changed, 95 insertions(+)
diff --git a/test/src/main/java/org/apache/accumulo/test/AuditMessageIT.java
b/test/src/main/java/org/apache/accumulo/test/AuditMessageIT.java
index cb88399c98..2e6ee5054a 100644
--- a/test/src/main/java/org/apache/accumulo/test/AuditMessageIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/AuditMessageIT.java
@@ -61,6 +61,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.hadoop.io.Text;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@@ -443,6 +444,7 @@ public class AuditMessageIT extends ConfigurableMacBase {
}
@Test
+ @Disabled // ELASTICITY_TODO
public void testDeniedAudits() throws AccumuloSecurityException,
AccumuloException,
[accumulo] 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 97f0c1bc77be44c32dd59234651564877f852722
Merge: 2d8bdf2b21 c14a97fe8c
Author: Dave Marion
AuthorDate: Tue Oct 24 14:25:20 2023 +
Merge branch 'main' into elasticity
core/src/main/java/org/apache/accumulo/core/conf/Property.java | 3 ---
1 file changed, 3 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index d1938b6281,903c5d7443..0b4df706b0
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -589,7 -573,12 +589,6 @@@ public enum Property
TSERV_COMPACTION_SERVICE_ROOT_PLANNER("tserver.compaction.major.service.root.planner",
DefaultCompactionPlanner.class.getName(), PropertyType.CLASSNAME,
"Compaction planner for root tablet service", "2.1.0"),
- // ELASTICITY_TODO: Deprecate TSERV_COMPACTION_SERVICE_ROOT_RATE_LIMIT in
3.x
- @Deprecated(since = "3.1", forRemoval = true)
-
TSERV_COMPACTION_SERVICE_ROOT_RATE_LIMIT("tserver.compaction.major.service.root.rate.limit",
"0B",
- PropertyType.BYTES,
- "Maximum number of bytes to read or write per second over all major"
- + " compactions in this compaction service, or 0B for unlimited.",
- "2.1.0"),
TSERV_COMPACTION_SERVICE_ROOT_MAX_OPEN(
"tserver.compaction.major.service.root.planner.opts.maxOpen", "30",
PropertyType.COUNT,
"The maximum number of files a compaction will open", "2.1.0"),
@@@ -602,7 -592,12 +601,6 @@@
TSERV_COMPACTION_SERVICE_META_PLANNER("tserver.compaction.major.service.meta.planner",
DefaultCompactionPlanner.class.getName(), PropertyType.CLASSNAME,
"Compaction planner for metadata table", "2.1.0"),
- // ELASTICITY_TODO: Deprecate TSERV_COMPACTION_SERVICE_META_RATE_LIMIT in
3.x
- @Deprecated(since = "3.1", forRemoval = true)
-
TSERV_COMPACTION_SERVICE_META_RATE_LIMIT("tserver.compaction.major.service.meta.rate.limit",
"0B",
- PropertyType.BYTES,
- "Maximum number of bytes to read or write per second over all major"
- + " compactions in this compaction service, or 0B for unlimited.",
- "2.1.0"),
TSERV_COMPACTION_SERVICE_META_MAX_OPEN(
"tserver.compaction.major.service.meta.planner.opts.maxOpen", "30",
PropertyType.COUNT,
"The maximum number of files a compaction will open", "2.1.0"),
@@@ -615,7 -611,12 +613,6 @@@
TSERV_COMPACTION_SERVICE_DEFAULT_PLANNER("tserver.compaction.major.service.default.planner",
DefaultCompactionPlanner.class.getName(), PropertyType.CLASSNAME,
"Planner for default compaction service.", "2.1.0"),
- // ELASTICITY_TODO: Deprecate TSERV_COMPACTION_SERVICE_DEFAULT_RATE_LIMIT
in 3.x
- @Deprecated(since = "3.1", forRemoval = true)
-
TSERV_COMPACTION_SERVICE_DEFAULT_RATE_LIMIT("tserver.compaction.major.service.default.rate.limit",
- "0B", PropertyType.BYTES,
- "Maximum number of bytes to read or write per second over all major"
- + " compactions in this compaction service, or 0B for unlimited.",
- "2.1.0"),
TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN(
"tserver.compaction.major.service.default.planner.opts.maxOpen", "10",
PropertyType.COUNT,
"The maximum number of files a compaction will open", "2.1.0"),
[accumulo] branch elasticity updated (2d8bdf2b21 -> 97f0c1bc77)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 2d8bdf2b21 Removed rate limiters as they weren't really being used (#3874) add c14a97fe8c Deprecate rate limiting properties (#3882) new 97f0c1bc77 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: core/src/main/java/org/apache/accumulo/core/conf/Property.java | 3 --- 1 file changed, 3 deletions(-)
[accumulo] branch main updated: Deprecate rate limiting properties (#3882)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new c14a97fe8c Deprecate rate limiting properties (#3882)
c14a97fe8c is described below
commit c14a97fe8ca91bd6bc5bba3a2f2ba1094f11fac6
Author: Dave Marion
AuthorDate: Tue Oct 24 10:15:36 2023 -0400
Deprecate rate limiting properties (#3882)
---
core/src/main/java/org/apache/accumulo/core/conf/Property.java | 3 +++
.../apache/accumulo/core/util/compaction/CompactionServicesConfig.java | 1 +
.../org/apache/accumulo/test/compaction/CompactionRateLimitingIT.java | 1 +
3 files changed, 5 insertions(+)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index a4b627068c..903c5d7443 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -573,6 +573,7 @@ public enum Property {
TSERV_COMPACTION_SERVICE_ROOT_PLANNER("tserver.compaction.major.service.root.planner",
DefaultCompactionPlanner.class.getName(), PropertyType.CLASSNAME,
"Compaction planner for root tablet service", "2.1.0"),
+ @Deprecated(since = "3.1", forRemoval = true)
TSERV_COMPACTION_SERVICE_ROOT_RATE_LIMIT("tserver.compaction.major.service.root.rate.limit",
"0B",
PropertyType.BYTES,
"Maximum number of bytes to read or write per second over all major"
@@ -591,6 +592,7 @@ public enum Property {
TSERV_COMPACTION_SERVICE_META_PLANNER("tserver.compaction.major.service.meta.planner",
DefaultCompactionPlanner.class.getName(), PropertyType.CLASSNAME,
"Compaction planner for metadata table", "2.1.0"),
+ @Deprecated(since = "3.1", forRemoval = true)
TSERV_COMPACTION_SERVICE_META_RATE_LIMIT("tserver.compaction.major.service.meta.rate.limit",
"0B",
PropertyType.BYTES,
"Maximum number of bytes to read or write per second over all major"
@@ -609,6 +611,7 @@ public enum Property {
TSERV_COMPACTION_SERVICE_DEFAULT_PLANNER("tserver.compaction.major.service.default.planner",
DefaultCompactionPlanner.class.getName(), PropertyType.CLASSNAME,
"Planner for default compaction service.", "2.1.0"),
+ @Deprecated(since = "3.1", forRemoval = true)
TSERV_COMPACTION_SERVICE_DEFAULT_RATE_LIMIT("tserver.compaction.major.service.default.rate.limit",
"0B", PropertyType.BYTES,
"Maximum number of bytes to read or write per second over all major"
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionServicesConfig.java
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionServicesConfig.java
index c6e1cea55d..b3f911c467 100644
---
a/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionServicesConfig.java
+++
b/core/src/main/java/org/apache/accumulo/core/util/compaction/CompactionServicesConfig.java
@@ -44,6 +44,7 @@ public class CompactionServicesConfig {
public static final CompactionServiceId DEFAULT_SERVICE =
CompactionServiceId.of("default");
+ @SuppressWarnings("removal")
private long getDefaultThroughput() {
return ConfigurationTypeHelper
.getMemoryAsBytes(Property.TSERV_COMPACTION_SERVICE_DEFAULT_RATE_LIMIT.getDefaultValue());
diff --git
a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionRateLimitingIT.java
b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionRateLimitingIT.java
index 0b4280f8d1..46e1060e24 100644
---
a/test/src/main/java/org/apache/accumulo/test/compaction/CompactionRateLimitingIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/CompactionRateLimitingIT.java
@@ -39,6 +39,7 @@ public class CompactionRateLimitingIT extends
ConfigurableMacBase {
public static final long BYTES_TO_WRITE = 10 * 1024 * 1024;
public static final long RATE = 1 * 1024 * 1024;
+ @SuppressWarnings("removal")
protected Property getThroughputProp() {
return Property.TSERV_COMPACTION_SERVICE_DEFAULT_RATE_LIMIT;
}
[accumulo] branch elasticity updated: Removed rate limiters as they weren't really being used (#3874)
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 2d8bdf2b21 Removed rate limiters as they weren't really being used
(#3874)
2d8bdf2b21 is described below
commit 2d8bdf2b217e5b42050af5625e201b3ac398e3d7
Author: Dave Marion
AuthorDate: Tue Oct 24 08:44:28 2023 -0400
Removed rate limiters as they weren't really being used (#3874)
Fixes #3597
---
.../accumulo/core/client/rfile/RFileScanner.java | 8 +-
.../org/apache/accumulo/core/conf/Property.java| 18 +-
.../apache/accumulo/core/file/FileOperations.java | 46 ++---
.../file/blockfile/impl/CachableBlockFile.java | 20 +--
.../accumulo/core/file/rfile/RFileOperations.java | 6 +-
.../accumulo/core/file/rfile/SplitLarge.java | 5 +-
.../accumulo/core/file/rfile/bcfile/BCFile.java| 24 ++-
.../core/file/streams/RateLimitedInputStream.java | 73
.../core/file/streams/RateLimitedOutputStream.java | 60 ---
.../accumulo/core/summary/SummaryReader.java | 6 +-
.../util/compaction/CompactionServicesConfig.java | 12 --
.../core/util/ratelimit/GuavaRateLimiter.java | 66 ---
.../core/util/ratelimit/NullRateLimiter.java | 37
.../accumulo/core/util/ratelimit/RateLimiter.java | 29
.../util/ratelimit/SharedRateLimiterFactory.java | 193 -
.../core/file/rfile/AbstractRFileTest.java | 2 +-
.../core/file/rfile/CreateCompatTestFile.java | 2 +-
.../core/file/rfile/MultiLevelIndexTest.java | 2 +-
.../core/file/rfile/MultiThreadedRFileTest.java| 2 +-
.../file/streams/RateLimitedInputStreamTest.java | 83 -
.../file/streams/RateLimitedOutputStreamTest.java | 68
.../accumulo/server/compaction/FileCompactor.java | 10 +-
.../org/apache/accumulo/compactor/ExtCEnv.java | 12 --
.../apache/accumulo/tserver/tablet/MinCEnv.java| 11 --
.../test/compaction/CompactionRateLimitingIT.java | 110
25 files changed, 47 insertions(+), 858 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java
b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java
index 79ea0bddfd..c5716fa6c8 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/rfile/RFileScanner.java
@@ -347,10 +347,10 @@ class RFileScanner extends ScannerOptions implements
Scanner {
for (int i = 0; i < sources.length; i++) {
// TODO may have been a bug with multiple files and caching in older
version...
-FSDataInputStream inputStream = (FSDataInputStream)
sources[i].getInputStream();
-CachableBuilder cb =
-new CachableBuilder().input(inputStream, "source-" +
i).length(sources[i].getLength())
-
.conf(opts.in.getConf()).cacheProvider(cacheProvider).cryptoService(cryptoService);
+CachableBuilder cb = new CachableBuilder()
+.input((FSDataInputStream) sources[i].getInputStream(), "source-"
+ i)
+
.length(sources[i].getLength()).conf(opts.in.getConf()).cacheProvider(cacheProvider)
+.cryptoService(cryptoService);
readers.add(RFile.getReader(cb, sources[i].getRange()));
}
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 53011b7f33..d1938b6281 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -589,11 +589,7 @@ public enum Property {
TSERV_COMPACTION_SERVICE_ROOT_PLANNER("tserver.compaction.major.service.root.planner",
DefaultCompactionPlanner.class.getName(), PropertyType.CLASSNAME,
"Compaction planner for root tablet service", "2.1.0"),
-
TSERV_COMPACTION_SERVICE_ROOT_RATE_LIMIT("tserver.compaction.major.service.root.rate.limit",
"0B",
- PropertyType.BYTES,
- "Maximum number of bytes to read or write per second over all major"
- + " compactions in this compaction service, or 0B for unlimited.",
- "2.1.0"),
+ // ELASTICITY_TODO: Deprecate TSERV_COMPACTION_SERVICE_ROOT_RATE_LIMIT in 3.x
TSERV_COMPACTION_SERVICE_ROOT_MAX_OPEN(
"tserver.compaction.major.service.root.planner.opts.maxOpen", "30",
PropertyType.COUNT,
"The maximum number of files a compaction will open", "2.1.0"),
@@ -606,11 +602,7 @@ public enum Property {
TSERV_COMPACTION_SERVICE_META_PLANNER("tserver.compaction.major.service.meta.planner",
DefaultCompactionPlanner.class.getName(), PropertyType.CLASSNAME,
"Compaction planner for metadata table", "2.1.0"),
-
