(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 220cdc54da5ba779675d635bcbcc02a6514cd5e6
Merge: 7baa5c76ab c88d368b2d
Author: Keith Turner
AuthorDate: Mon May 13 17:35:55 2024 -0400
Merge branch '2.1'
.../accumulo/tserver/tablet/CompactableImpl.java | 31 +--
.../org/apache/accumulo/tserver/tablet/Tablet.java | 8 +++
.../tablet/CompactableImplFileManagerTest.java | 61 ++
3 files changed, 86 insertions(+), 14 deletions(-)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index d519ffd172,37b23dc6c4..c17f808ab2
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@@ -232,15 -247,32 +232,33 @@@ public class CompactableImpl implement
protected abstract long getNanoTime();
- boolean initiateSelection(CompactionKind kind) {
+ /**
+ * @return the last id of the last successful user compaction
+ */
+ protected abstract long getLastCompactId();
+
+ boolean initiateSelection(CompactionKind kind, Long compactionId) {
- Preconditions
- .checkArgument(kind == DeprecatedCompactionKind.SELECTOR || kind ==
CompactionKind.USER);
+ Preconditions.checkArgument(
- kind == CompactionKind.SELECTOR && compactionId == null
++ kind == DeprecatedCompactionKind.SELECTOR && compactionId == null
+ || kind == CompactionKind.USER && compactionId != null,
+ "Unexpected kind and/or compaction id: %s %s", kind, compactionId);
- if (selectStatus == FileSelectionStatus.NOT_ACTIVE || (kind ==
CompactionKind.USER
- && selectKind == CompactionKind.SELECTOR &&
noneRunning(CompactionKind.SELECTOR)
- && selectStatus != FileSelectionStatus.SELECTING)) {
+ if (selectStatus == FileSelectionStatus.NOT_ACTIVE
+ || (kind == CompactionKind.USER && selectKind ==
DeprecatedCompactionKind.SELECTOR
+ && noneRunning(DeprecatedCompactionKind.SELECTOR)
+ && selectStatus != FileSelectionStatus.SELECTING)) {
+
+ // Check compaction id when a lock is held and no other user
compactions have files
+ // selected, at this point the results of any previous user
compactions should be seen. If
+ // user compaction is currently running, then will not get this far
because of the checks a
+ // few lines up.
+ if (kind == CompactionKind.USER && getLastCompactId() >=
compactionId) {
+ // This user compaction has already completed, so no need to
initiate selection of files
+ // for user compaction.
+ return false;
+ }
+
selectStatus = FileSelectionStatus.NEW;
selectKind = kind;
selectedFiles.clear();
diff --cc
server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/CompactableImplFileManagerTest.java
index 35d5119dfa,743ac8feca..2f04a875f2
---
a/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/CompactableImplFileManagerTest.java
+++
b/server/tserver/src/test/java/org/apache/accumulo/tserver/tablet/CompactableImplFileManagerTest.java
@@@ -28,8 -30,10 +29,9 @@@ import static org.junit.jupiter.api.Ass
import java.time.Duration;
import java.util.HashSet;
-import java.util.List;
import java.util.Optional;
import java.util.Set;
+ import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@@ -111,10 -117,10 +113,10 @@@ public class CompactableImplFileManager
assertFalse(fileMgr.reserveFiles(staleJob));
assertEquals(newFiles("F0.rf", "F1.rf", "F2.rf", "F3.rf"),
-fileMgr.getCandidates(tabletFiles, SYSTEM, false));
-assertNoCandidates(fileMgr, tabletFiles, CHOP, USER, SELECTOR);
+fileMgr.getCandidates(tabletFiles, SYSTEM));
+assertNoCandidates(fileMgr, tabletFiles, USER, SELECTOR);
- assertTrue(fileMgr.initiateSelection(USER));
+ assertTrue(fileMgr.initiateSelection(USER, 1L));
assertFalse(fileMgr.reserveFiles(staleJob));
@@@ -372,6 -380,96 +374,39 @@@
}
- @Test
- public void testChop() {
-TestFileManager fileMgr = new TestFileManager();
-
-// simulate a compaction because files that were created by compaction
are remembered as not
-// needing a chop
-var job1 = newJob(SYSTEM, "F0.rf", "F1.rf");
-assertTrue(fileMgr.reserveFiles(job1));
-fileMgr.completed(job1, newFile("C5.rf"));
-
-var tabletFiles = newFiles("C5.rf", "F2.rf", "F3.rf",
"F4.rf");
-
-ChopSelector chopSel = fileMgr.initiateChop(tabletFiles);
-assertEquals(ChopSelectionStatus.SELECTING, fileMgr.getChopStatus());
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 7baa5c76abe1dc459d6d554bd5d19cf35c899756 Merge: 861d28b64b d4e2ae729a Author: Keith Turner AuthorDate: Mon May 13 10:48:03 2024 -0400 Merge branch '2.1' .../main/java/org/apache/accumulo/core/clientImpl/ClientContext.java | 4 ++-- .../apache/accumulo/core/spi/scan/ConfigurableScanServerSelector.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --cc core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java index acc06022e8,e4ba0028c9..093aa7f04e --- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java +++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientContext.java @@@ -19,8 -19,10 +19,9 @@@ package org.apache.accumulo.core.clientImpl; import static com.google.common.base.Preconditions.checkArgument; + import static com.google.common.base.Suppliers.memoize; import static com.google.common.base.Suppliers.memoizeWithExpiration; import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.Objects.requireNonNull; import static java.util.concurrent.TimeUnit.MILLISECONDS; import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION;
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit f4c1f2465886c5a5f6ebdc384b0a695ad0ac56ec Merge: 33e2d055af 86c81e42d2 Author: Dave Marion AuthorDate: Tue May 7 20:33:21 2024 + Merge branch '2.1' .../main/java/org/apache/accumulo/tserver/ScanServer.java | 13 - 1 file changed, 4 insertions(+), 9 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 8da73ce467f9311d247ac1eaa8062a5f62853276 Merge: 09adb11491 61ca7d8070 Author: Keith Turner AuthorDate: Wed May 1 22:59:21 2024 + Merge branch '2.1' .../coordinator/CompactionCoordinator.java | 8 +++-- .../coordinator/DeadCompactionDetector.java| 11 +-- .../accumulo/manager/TabletGroupWatcher.java | 35 ++ 3 files changed, 38 insertions(+), 16 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 94b8d96e5c334ee3c7adbf665b9ddd9e1f46a692
Merge: 6c1e453f53 6d0b476852
Author: Christopher Tubbs
AuthorDate: Wed Apr 24 17:10:59 2024 -0400
Merge branch '2.1'
Merge of 2.1 to main branch to include all changes up through #4459
.../org/apache/accumulo/core/conf/Property.java| 24 +++---
.../accumulo/core/iterators/user/GrepIterator.java | 2 +-
.../core/iterators/user/GrepIteratorTest.java | 18
.../java/org/apache/accumulo/manager/Manager.java | 14 -
.../accumulo/tserver/tablet/CompactableImpl.java | 13 ++--
.../accumulo/shell/commands/GrepCommand.java | 2 +-
.../org/apache/accumulo/test/shell/ShellIT.java| 1 +
7 files changed, 62 insertions(+), 12 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 6dc56205ba,80a9b21b33..463b37cbcf
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -1533,13 -1886,15 +1550,14 @@@ public enum Property
// white list prefixes
return key.startsWith(Property.TABLE_PREFIX.getKey())
|| key.startsWith(Property.TSERV_PREFIX.getKey())
+|| key.startsWith(Property.COMPACTION_SERVICE_PREFIX.getKey())
+ || key.startsWith(Property.SSERV_PREFIX.getKey())
|| key.startsWith(Property.MANAGER_PREFIX.getKey())
-|| key.startsWith(Property.MASTER_PREFIX.getKey())
|| key.startsWith(Property.GC_PREFIX.getKey())
|| key.startsWith(Property.GENERAL_ARBITRARY_PROP_PREFIX.getKey())
+|| key.equals(Property.COMPACTION_WARN_TIME.getKey())
||
key.equals(Property.GENERAL_FILE_NAME_ALLOCATION_BATCH_SIZE_MIN.getKey())
-||
key.equals(Property.GENERAL_FILE_NAME_ALLOCATION_BATCH_SIZE_MAX.getKey())
-|| key.startsWith(VFS_CONTEXT_CLASSPATH_PROPERTY.getKey())
-|| key.startsWith(REPLICATION_PREFIX.getKey());
+||
key.equals(Property.GENERAL_FILE_NAME_ALLOCATION_BATCH_SIZE_MAX.getKey());
}
/**
diff --cc server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index 9cead258f7,86a1dd71d3..5c6cc2f436
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@@ -90,12 -85,16 +90,13 @@@ import org.apache.accumulo.core.manager
import org.apache.accumulo.core.manager.thrift.ManagerGoalState;
import org.apache.accumulo.core.manager.thrift.ManagerMonitorInfo;
import org.apache.accumulo.core.manager.thrift.ManagerState;
-import org.apache.accumulo.core.master.thrift.BulkImportState;
-import org.apache.accumulo.core.master.thrift.TableInfo;
-import org.apache.accumulo.core.master.thrift.TabletServerStatus;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.manager.thrift.TableInfo;
+import org.apache.accumulo.core.manager.thrift.TabletServerStatus;
+import org.apache.accumulo.core.metadata.AccumuloTable;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.TabletLocationState;
-import org.apache.accumulo.core.metadata.TabletState;
import org.apache.accumulo.core.metadata.schema.Ample.DataLevel;
+ import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
import org.apache.accumulo.core.metrics.MetricsInfo;
import org.apache.accumulo.core.metrics.MetricsProducer;
@@@ -703,17 -709,19 +704,20 @@@ public class Manager extends AbstractSe
*/
private void cleanupNonexistentMigrations(final AccumuloClient
accumuloClient)
throws TableNotFoundException {
- Scanner scanner = accumuloClient.createScanner(MetadataTable.NAME,
Authorizations.EMPTY);
+ Scanner scanner =
+ accumuloClient.createScanner(AccumuloTable.METADATA.tableName(),
Authorizations.EMPTY);
TabletColumnFamily.PREV_ROW_COLUMN.fetch(scanner);
- Set found = new HashSet<>();
+ scanner.setRange(MetadataSchema.TabletsSection.getRange());
+ Set notSeen;
+ synchronized (migrations) {
+ notSeen = new HashSet<>(migrations.keySet());
+ }
for (Entry entry : scanner) {
KeyExtent extent = KeyExtent.fromMetaPrevRow(entry);
- if (migrations.containsKey(extent)) {
- found.add(extent);
- }
+ notSeen.remove(extent);
}
- migrations.keySet().retainAll(found);
+ // remove tablets that used to be in migrations and were not seen in
the metadata table
+ migrations.keySet().removeAll(notSeen);
}
/**
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 859694aba88b64ce3f9c5559c0f772f987dbb0e4 Merge: c39599d75d 7c6572caf3 Author: Keith Turner AuthorDate: Thu Apr 11 19:22:12 2024 -0400 Merge branch '2.1' assemble/bin/accumulo-cluster | 2 +- .../apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 84e8557ab8ddabd23c8813d9dbed2e1223d99646
Merge: b912506d43 90bcb465fe
Author: Christopher L. Shannon
AuthorDate: Sat Apr 6 15:23:23 2024 -0400
Merge branch '2.1'
.../org/apache/accumulo/core/metadata/TabletLocationState.java| 3 ++-
.../org/apache/accumulo/core/metadata/schema/TabletMetadata.java | 4 +++-
.../apache/accumulo/core/metadata/schema/TabletMetadataTest.java | 8
3 files changed, 13 insertions(+), 2 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/TabletLocationState.java
index eeaa5cde95,0a70594027..4c2421b4e8
---
a/core/src/main/java/org/apache/accumulo/core/metadata/TabletLocationState.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/TabletLocationState.java
@@@ -57,8 -57,9 +58,8 @@@ public class TabletLocationState
}
public TabletLocationState(KeyExtent extent, Location future, Location
current, Location last,
- SuspendingTServer suspend, Collection> walogs,
boolean chopped)
- throws BadLocationStateException {
+ SuspendingTServer suspend, Collection walogs) throws
BadLocationStateException {
- this.extent = extent;
+ this.extent = Objects.requireNonNull(extent);
this.future = validateLocation(future,
TabletMetadata.LocationType.FUTURE);
this.current = validateLocation(current,
TabletMetadata.LocationType.CURRENT);
this.last = validateLocation(last, TabletMetadata.LocationType.LAST);
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index af31605d51,c1e816109a..b98d9e8540
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@@ -371,7 -373,9 +371,9 @@@ public class TabletMetadata
future = location;
}
// only care about the state so don't need walogs and chopped params
- var tls = new TabletLocationState(extent, future, current, last,
suspend, null);
+ // Use getExtent() when passing the extent as the private reference may
not have been
+ // initialized yet. This will also ensure PREV_ROW was fetched
- var tls = new TabletLocationState(getExtent(), future, current, last,
suspend, null, false);
++ var tls = new TabletLocationState(getExtent(), future, current, last,
suspend, null);
return tls.getState(liveTServers);
} catch (TabletLocationState.BadLocationStateException blse) {
throw new IllegalArgumentException("Error creating
TabletLocationState", blse);
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit c4ceb7e7a9f183059d753386d6a09cd880f1bb08 Merge: 5d741657e7 01ccb96472 Author: Daniel Roberts AuthorDate: Mon Apr 1 15:25:19 2024 + Merge branch '2.1' pom.xml | 2 +- .../apache/accumulo/shell/commands/HistoryCommandTest.java| 11 ++- 2 files changed, 7 insertions(+), 6 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit a3a6fc2634d2d067d4c951f441bcebb2a06ed7b0 Merge: eae367264b 33894e6997 Author: Dave Marion AuthorDate: Fri Mar 22 12:28:06 2024 + Merge branch '2.1' .../main/java/org/apache/accumulo/server/rpc/TServerUtils.java | 9 + 1 file changed, 9 insertions(+) diff --cc server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java index 5f0cebd05a,9f37990926..0f28ecab8b --- a/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java +++ b/server/base/src/main/java/org/apache/accumulo/server/rpc/TServerUtils.java @@@ -46,7 -46,9 +46,8 @@@ import org.apache.accumulo.core.rpc.Ssl import org.apache.accumulo.core.rpc.ThriftUtil; import org.apache.accumulo.core.rpc.UGIAssumingTransportFactory; import org.apache.accumulo.core.util.Halt; -import org.apache.accumulo.core.util.HostAndPort; import org.apache.accumulo.core.util.Pair; + import org.apache.accumulo.core.util.UtilWaitThread; import org.apache.accumulo.core.util.threads.ThreadPools; import org.apache.accumulo.core.util.threads.Threads; import org.apache.accumulo.server.ServerContext; @@@ -70,7 -72,7 +71,8 @@@ import org.apache.thrift.transport.TTra import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import com.google.common.base.Preconditions; +import com.google.common.net.HostAndPort; import com.google.common.primitives.Ints; /**
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit eae367264b0e224c59de90ba72fcae463f348357 Merge: eaa1f678c9 6784e59da3 Author: Christopher L. Shannon AuthorDate: Fri Mar 22 07:47:34 2024 -0400 Merge branch '2.1' .../main/java/org/apache/accumulo/core/conf/PropertyType.java | 2 +- pom.xml| 10 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --cc pom.xml index ff26ae9641,fb747c50a8..275fc88b18 --- a/pom.xml +++ b/pom.xml @@@ -429,8 -441,19 +429,8 @@@ org.apache.commons commons-text - 1.10.0 + 1.11.0 - -org.apache.commons -commons-vfs2 -2.9.0 - - -org.apache.hadoop -hadoop-hdfs-client - - - org.apache.curator curator-framework
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit eaa1f678c96b135534fba89fe585107c4f242e54
Merge: 0ad96b1dc0 7947c2c2df
Author: Keith Turner
AuthorDate: Wed Mar 20 18:39:26 2024 -0400
Merge branch '2.1'
.../core/client/ClientSideIteratorScanner.java | 39 +++-
.../accumulo/core/clientImpl/OfflineIterator.java | 27 +-
.../accumulo/core/clientImpl/ScannerImpl.java | 10 ++
.../apache/accumulo/test/ClientSideIteratorIT.java | 108 +
4 files changed, 178 insertions(+), 6 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
index a50561263f,362c4e85d6..7e300e917d
---
a/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/ClientSideIteratorScanner.java
@@@ -122,6 -133,22 +133,16 @@@ public class ClientSideIteratorScanner
public SamplerConfiguration getSamplerConfiguration() {
return samplerConfig;
}
+
-@Deprecated(since = "2.1.0")
-@Override
-public ServiceEnvironment getServiceEnv() {
- return new ClientServiceEnvironmentImpl(context.get());
-}
-
+ @Override
+ public PluginEnvironment getPluginEnv() {
+ return new ClientServiceEnvironmentImpl(context.get());
+ }
+
+ @Override
+ public TableId getTableId() {
+ return tableId.get();
+ }
}
/**
diff --cc
core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java
index 0e7a3fd4d2,a03cc811ab..9200591ff2
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/OfflineIterator.java
@@@ -135,7 -153,24 +140,18 @@@ class OfflineIterator implements Iterat
if (sampleConf == null) {
throw new SampleNotPresentException();
}
- return new OfflineIteratorEnvironment(authorizations, conf, true,
sampleConf);
+ return new OfflineIteratorEnvironment(context, tableId, authorizations,
conf, true,
+ sampleConf);
+ }
+
-@Deprecated(since = "2.1.0")
-@Override
-public ServiceEnvironment getServiceEnv() {
- return new ClientServiceEnvironmentImpl(context);
-}
-
+ @Override
+ public PluginEnvironment getPluginEnv() {
+ return new ClientServiceEnvironmentImpl(context);
+ }
+
+ @Override
+ public TableId getTableId() {
+ return tableId;
}
}
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit da0e0e6624eef8c207022636a67a41bdd848fa49
Merge: a3616bf46f f3e75f0e46
Author: Christopher Tubbs
AuthorDate: Fri Mar 15 18:04:39 2024 -0400
Merge branch '2.1'
.../org/apache/accumulo/compactor/Compactor.java | 44 +-
1 file changed, 10 insertions(+), 34 deletions(-)
diff --cc
server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index b81c30dee3,5e09e48fdf..f4affca907
---
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@@ -106,8 -105,6 +106,7 @@@ import org.apache.accumulo.server.fs.Vo
import org.apache.accumulo.server.rpc.ServerAddress;
import org.apache.accumulo.server.rpc.TServerUtils;
import org.apache.accumulo.server.rpc.ThriftProcessorTypes;
- import org.apache.accumulo.server.security.SecurityOperation;
+import org.apache.accumulo.server.zookeeper.TransactionWatcher;
import org.apache.hadoop.fs.Path;
import org.apache.thrift.TException;
import org.apache.thrift.transport.TTransportException;
@@@ -130,42 -139,23 +129,23 @@@ public class Compactor extends Abstract
protected static final CompactionJobHolder JOB_HOLDER = new
CompactionJobHolder();
- private final GarbageCollectionLogger gcLogger = new
GarbageCollectionLogger();
private final UUID compactorId = UUID.randomUUID();
- private final AccumuloConfiguration aconf;
private final String queueName;
protected final AtomicReference currentCompactionId =
new AtomicReference<>();
- private final CompactionWatcher watcher;
- private SecurityOperation security;
private ServiceLock compactorLock;
private ServerAddress compactorAddress = null;
+ private PausedCompactionMetrics pausedMetrics;
// Exposed for tests
protected volatile boolean shutdown = false;
private final AtomicBoolean compactionRunning = new AtomicBoolean(false);
- protected Compactor(CompactorServerOpts opts, String[] args) {
+ protected Compactor(ConfigOpts opts, String[] args) {
- this(opts, args, null);
- }
-
- protected Compactor(ConfigOpts opts, String[] args, AccumuloConfiguration
conf) {
super("compactor", opts, args);
- aconf = conf == null ? super.getConfiguration() : conf;
- queueName = aconf.get(Property.COMPACTOR_QUEUE_NAME);
- setupSecurity();
- watcher = new CompactionWatcher(aconf);
- var schedExecutor =
-
ThreadPools.getServerThreadPools().createGeneralScheduledExecutorService(aconf);
- startCancelChecker(schedExecutor, TIME_BETWEEN_CANCEL_CHECKS);
- printStartupMsg();
- }
-
- @Override
- public AccumuloConfiguration getConfiguration() {
- return aconf;
-queueName = opts.getQueueName();
++queueName = super.getConfiguration().get(Property.COMPACTOR_QUEUE_NAME);
}
@Override
@@@ -176,10 -165,13 +156,6 @@@
CompactionWatcher.setTimer(timer);
}
- protected void setupSecurity() {
- security = getContext().getSecurityOperation();
- protected void startGCLogger(ScheduledThreadPoolExecutor schedExecutor) {
-ScheduledFuture future =
-schedExecutor.scheduleWithFixedDelay(() ->
gcLogger.logGCInfo(getConfiguration()), 0,
-TIME_BETWEEN_GC_CHECKS, TimeUnit.MILLISECONDS);
-ThreadPools.watchNonCriticalScheduledTask(future);
-- }
--
protected void startCancelChecker(ScheduledThreadPoolExecutor schedExecutor,
long timeBetweenChecks) {
ThreadPools.watchCriticalScheduledTask(schedExecutor.scheduleWithFixedDelay(
@@@ -625,7 -601,14 +596,12 @@@
| SecurityException e1) {
LOG.error("Error initializing metrics, metrics will not be emitted.",
e1);
}
-MetricsUtil.initializeProducers(this);
+ var watcher = new CompactionWatcher(getConfiguration());
+ var schedExecutor = ThreadPools.getServerThreadPools()
+ .createGeneralScheduledExecutorService(getConfiguration());
-startGCLogger(schedExecutor);
+ startCancelChecker(schedExecutor, TIME_BETWEEN_CANCEL_CHECKS);
+
LOG.info("Compactor started, waiting for work");
try {
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit fe552af0dd23d263aca96dd6d356df7070641b29 Merge: 9839e4d42f 7c9204274c Author: Dave Marion AuthorDate: Fri Mar 15 17:22:56 2024 + Merge branch '2.1' .../apache/accumulo/core/util/compaction/RunningCompaction.java | 8 ++-- 1 file changed, 6 insertions(+), 2 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 9839e4d42fae9bd20b1529ca0e5e2fed4122
Merge: 23e17129de 05c2f45042
Author: Dave Marion
AuthorDate: Wed Mar 13 19:52:45 2024 +
Merge branch '2.1'
.../coordinator/CompactionCoordinator.java | 32 --
.../accumulo/coordinator/QueueSummaries.java | 8 ++
2 files changed, 37 insertions(+), 3 deletions(-)
diff --cc
server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index d5ecbf9ddd,b0ec498a9e..685080c5b1
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@@ -22,8 -23,7 +22,9 @@@ import static com.google.common.util.co
import java.lang.reflect.InvocationTargetException;
import java.net.UnknownHostException;
+import java.util.ArrayList;
+ import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@@ -321,35 -325,58 +325,57 @@@ public class CompactionCoordinator exte
LOG.info("Shutting down");
}
+ private Map> getIdleCompactors() {
+
+ Map> allCompactors =
+ ExternalCompactionUtil.getCompactorAddrs(getContext());
+
+ Set emptyQueues = new HashSet<>();
+
+ // Remove all of the compactors that are running a compaction
+ RUNNING_CACHE.values().forEach(rc -> {
+ List busyCompactors = allCompactors.get(rc.getQueueName());
+ if (busyCompactors != null
+ &&
busyCompactors.remove(HostAndPort.fromString(rc.getCompactorAddress( {
+ if (busyCompactors.isEmpty()) {
+ emptyQueues.add(rc.getQueueName());
+ }
+ }
+ });
+ // Remove entries with empty queues
+ emptyQueues.forEach(e -> allCompactors.remove(e));
+ return allCompactors;
+ }
+
private void updateSummaries() {
-ExecutorService executor =
ThreadPools.getServerThreadPools().createFixedThreadPool(10,
-"Compaction Summary Gatherer", false);
-try {
- Set queuesSeen = new ConcurrentSkipListSet<>();
- tserverSet.getCurrentServers().forEach(tsi -> {
-executor.execute(() -> updateSummaries(tsi, queuesSeen));
- });
+final ArrayList> tasks = new ArrayList<>();
+Set queuesSeen = new ConcurrentSkipListSet<>();
- executor.shutdown();
+tserverSet.getCurrentServers().forEach(tsi -> {
+ tasks.add(summariesExecutor.submit(() -> updateSummaries(tsi,
queuesSeen)));
+});
- try {
-while (!executor.awaitTermination(1, TimeUnit.MINUTES)) {}
- } catch (InterruptedException e) {
-Thread.currentThread().interrupt();
-throw new RuntimeException(e);
+// Wait for all tasks to complete
+while (!tasks.isEmpty()) {
+ Iterator> iter = tasks.iterator();
+ while (iter.hasNext()) {
+Future f = iter.next();
+if (f.isDone()) {
+ iter.remove();
+}
}
+ Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
+}
- // remove any queues that were seen in the past, but were not seen in
the latest gathering of
- // summaries
- TIME_COMPACTOR_LAST_CHECKED.keySet().retainAll(queuesSeen);
+// remove any queues that were seen in the past, but were not seen in the
latest gathering of
+// summaries
+TIME_COMPACTOR_LAST_CHECKED.keySet().retainAll(queuesSeen);
- // add any queues that were never seen before
- queuesSeen.forEach(q -> {
-TIME_COMPACTOR_LAST_CHECKED.computeIfAbsent(q, k ->
System.currentTimeMillis());
- });
-} finally {
- executor.shutdownNow();
-}
+// add any queues that were never seen before
+queuesSeen.forEach(q -> {
+ TIME_COMPACTOR_LAST_CHECKED.computeIfAbsent(q, k ->
System.currentTimeMillis());
+});
}
private void updateSummaries(TServerInstance tsi, Set queuesSeen) {
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 171a1e144caa9d2eb382846d2dc45a424487d3ec
Merge: 162b8effb1 92331ea113
Author: Daniel Roberts
AuthorDate: Tue Mar 12 15:17:40 2024 +
Merge branch '2.1'
.../accumulo/core/file/rfile/GenerateSplits.java | 32 +--
.../core/file/rfile/GenerateSplitsTest.java| 64 +++---
2 files changed, 82 insertions(+), 14 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/file/rfile/GenerateSplits.java
index 5928ade21d,865210a970..7ee151f0a2
--- a/core/src/main/java/org/apache/accumulo/core/file/rfile/GenerateSplits.java
+++ b/core/src/main/java/org/apache/accumulo/core/file/rfile/GenerateSplits.java
@@@ -137,19 -143,33 +144,33 @@@ public class GenerateSplits implements
Path path = new Path(file);
fs = PrintInfo.resolveFS(log, hadoopConf, path);
// get all the files in the directory
- filePaths.addAll(getFiles(fs, path));
+ files.addAll(getFiles(fs, path));
}
-if (filePaths.isEmpty()) {
+if (files.isEmpty()) {
throw new IllegalArgumentException("No files were found in " +
opts.files);
} else {
- log.trace("Found the following files: {}", filePaths);
+ log.trace("Found the following files: {}", files);
}
+ if (!encode) {
+ // Generate the allowed Character set
+ for (int i = 0; i < 10; i++) {
+ // 0-9
+ allowedChars.add((char) (i + 48));
+ }
+ for (int i = 0; i < 26; i++) {
+ // Uppercase A-Z
+ allowedChars.add((char) (i + 65));
+ // Lowercase a-z
+ allowedChars.add((char) (i + 97));
+ }
+ }
+
// if no size specified look at indexed keys first
if (opts.splitSize == 0) {
- splits = getIndexKeys(siteConf, hadoopConf, fs, filePaths,
requestedNumSplits, encode,
- cryptoService);
+ splits =
+ getIndexKeys(siteConf, hadoopConf, fs, files, requestedNumSplits,
encode, cryptoService);
// if there weren't enough splits indexed, try again with size = 0
if (splits.size() < requestedNumSplits) {
log.info("Only found {} indexed keys but need {}. Doing a full scan
on files {}",
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit e3039f402bcc725dbf48e18a22f69267323935f1
Merge: 9d9bcc1d48 802b6e948d
Author: Christopher L. Shannon
AuthorDate: Mon Mar 11 11:16:11 2024 -0400
Merge branch '2.1'
.../core/metadata/schema/TabletsMetadata.java | 2 +-
.../core/metadata/schema/TabletMetadataTest.java | 25 ++
2 files changed, 26 insertions(+), 1 deletion(-)
diff --cc
core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
index 5287d689ba,67a22ccd3f..9a352b82d5
---
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
@@@ -41,7 -41,8 +43,9 @@@ import java.util.Map
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
+import java.util.UUID;
+ import java.util.concurrent.atomic.AtomicBoolean;
+ import java.util.stream.Stream;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
@@@ -259,30 -259,27 +263,51 @@@ public class TabletMetadataTest
assertFalse(tm.hasCurrent());
}
+ @Test
+ public void testMergedColumn() {
+KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new
Text("da"));
+
+// Test merged column set
+Mutation mutation = TabletColumnFamily.createPrevRowMutation(extent);
+MERGED_COLUMN.put(mutation, MERGED_VALUE);
+TabletMetadata tm =
TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+EnumSet.of(ColumnType.MERGED), true);
+assertTrue(tm.hasMerged());
+
+// Column not set
+mutation = TabletColumnFamily.createPrevRowMutation(extent);
+tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+EnumSet.of(ColumnType.MERGED), true);
+assertFalse(tm.hasMerged());
+
+// MERGED Column not fetched
+mutation = TabletColumnFamily.createPrevRowMutation(extent);
+tm = TabletMetadata.convertRow(toRowMap(mutation).entrySet().iterator(),
+EnumSet.of(ColumnType.PREV_ROW), true);
+assertThrows(IllegalStateException.class, tm::hasMerged);
+ }
+
+ @Test
+ public void testTabletsMetadataAutoClose() throws Exception {
+ AtomicBoolean closeCalled = new AtomicBoolean();
+ AutoCloseable autoCloseable = () -> closeCalled.set(true);
+ Constructor tmConstructor =
+ TabletsMetadata.class.getDeclaredConstructor(AutoCloseable.class,
Iterable.class);
+ tmConstructor.setAccessible(true);
+
+ try (TabletsMetadata ignored = tmConstructor.newInstance(autoCloseable,
List.of())) {
+ // test autoCloseable used directly on TabletsMetadata
+ }
+ assertTrue(closeCalled.get());
+
+ closeCalled.set(false);
+ try (Stream ignored =
+ tmConstructor.newInstance(autoCloseable, List.of()).stream()) {
+ // test stream delegates to close on TabletsMetadata
+ }
+ assertTrue(closeCalled.get());
+ }
+
private SortedMap toRowMap(Mutation mutation) {
SortedMap rowMap = new TreeMap<>();
mutation.getUpdates().forEach(cu -> {
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit f3d5fb01d701a6932d37d2f67f52cc0eefa64d50
Merge: ebf7054d1f d91d016211
Author: Dave Marion
AuthorDate: Mon Mar 4 18:14:16 2024 +
Merge branch '2.1'
.../core/clientImpl/ThriftTransportKey.java| 29 --
.../core/clientImpl/ThriftTransportPool.java | 110 +
.../org/apache/accumulo/core/rpc/ThriftUtil.java | 7 +-
.../accumulo/core/rpc/clients/TServerClient.java | 105 +++-
.../core/rpc/clients/ThriftClientTypes.java| 6 +-
.../core/clientImpl/ThriftTransportKeyTest.java| 25 ++---
.../coordinator/CompactionCoordinator.java | 4 +-
.../apache/accumulo/test/TransportCachingIT.java | 42
8 files changed, 158 insertions(+), 170 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportKey.java
index f332a09492,f4c7047d6d..0f84154a15
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportKey.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportKey.java
@@@ -24,9 -24,10 +24,10 @@@ import java.util.Objects
import org.apache.accumulo.core.rpc.SaslConnectionParams;
import org.apache.accumulo.core.rpc.SslConnectionParams;
+ import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
-import org.apache.accumulo.core.util.HostAndPort;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.net.HostAndPort;
@VisibleForTesting
public class ThriftTransportKey {
@@@ -54,12 -58,18 +58,18 @@@
this.saslParams = saslParams;
if (saslParams != null && sslParams != null) {
// TSasl and TSSL transport factories don't play nicely together
- throw new RuntimeException("Cannot use both SSL and SASL thrift
transports");
+ throw new IllegalArgumentException("Cannot use both SSL and SASL thrift
transports");
}
- this.hash = Objects.hash(server, timeout, sslParams, saslParams);
+ this.hash = Objects.hash(type, server, timeout, sslParams, saslParams);
}
- HostAndPort getServer() {
+ @VisibleForTesting
+ public ThriftClientTypes getType() {
+ return type;
+ }
+
+ @VisibleForTesting
+ public HostAndPort getServer() {
return server;
}
diff --cc
core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportPool.java
index d1bc17e945,a3d38aa10a..b3f205fa2a
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportPool.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftTransportPool.java
@@@ -41,6 -41,8 +41,7 @@@ import java.util.function.LongSupplier
import java.util.function.Supplier;
import org.apache.accumulo.core.rpc.ThriftUtil;
+ import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
-import org.apache.accumulo.core.util.HostAndPort;
import org.apache.accumulo.core.util.Pair;
import org.apache.accumulo.core.util.threads.Threads;
import org.apache.thrift.TConfiguration;
@@@ -49,9 -51,7 +50,8 @@@ import org.apache.thrift.transport.TTra
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
- import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.net.HostAndPort;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@@@ -109,71 -110,40 +109,40 @@@ public class ThriftTransportPool
return pool;
}
- public TTransport getTransport(HostAndPort location, long milliseconds,
ClientContext context)
- throws TTransportException {
- ThriftTransportKey cacheKey = new ThriftTransportKey(location,
milliseconds, context);
+ public TTransport getTransport(ThriftClientTypes type, HostAndPort
location, long milliseconds,
+ ClientContext context, boolean preferCached) throws TTransportException
{
- CachedConnection connection = connectionPool.reserveAny(cacheKey);
-
- if (connection != null) {
- log.trace("Using existing connection to {}", cacheKey.getServer());
- return connection.transport;
- } else {
- return createNewTransport(cacheKey);
+ ThriftTransportKey cacheKey = new ThriftTransportKey(type, location,
milliseconds, context);
+ if (preferCached) {
+ CachedConnection connection = connectionPool.reserveAny(cacheKey);
+ if (connection != null) {
+ log.trace("Using existing connection to {}", cacheKey.getServer());
+ return connection.transport;
+ }
}
+ return createNewTransport(cacheKey);
}
- @VisibleForTesting
- public Pair getAnyTransport(List
servers,
- boolean preferCachedConnection) throws TTransportException {
-
- servers = new ArrayList<>(servers);
-
- if (preferCachedConnection) {
- HashSet serversSet = new
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit ebf7054d1f6bf62d5170dcbbb07c7b0ee572cf2c Merge: c976af383f 18b745466e Author: Dave Marion AuthorDate: Mon Mar 4 12:34:51 2024 + Merge branch '2.1' assemble/bin/accumulo-cluster | 10 +- .../accumulo/core/conf/cluster/ClusterConfigParser.java | 5 - .../accumulo/core/conf/cluster/ClusterConfigParserTest.java | 11 --- .../core/conf/cluster/cluster-with-optional-services.yaml | 3 ++- 4 files changed, 23 insertions(+), 6 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit c976af383f992bdefcdd579220c704235960a6c7
Merge: 9817474333 507a96380b
Author: Christopher L. Shannon
AuthorDate: Fri Mar 1 17:21:36 2024 -0500
Merge branch '2.1'
.../accumulo/test/functional/MetadataIT.java | 71 ++
1 file changed, 71 insertions(+)
diff --cc test/src/main/java/org/apache/accumulo/test/functional/MetadataIT.java
index 0b3116c395,3c6514624e..60ca6cb562
--- a/test/src/main/java/org/apache/accumulo/test/functional/MetadataIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MetadataIT.java
@@@ -45,7 -48,10 +47,9 @@@ import org.apache.accumulo.core.data.Ke
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
+ import org.apache.accumulo.core.iterators.user.VersioningIterator;
-import org.apache.accumulo.core.metadata.MetadataTable;
-import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.AccumuloTable;
+ import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.DeletesSection;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
@@@ -221,4 -224,67 +227,69 @@@ public class MetadataIT extends Accumul
}
}
+
+ // Test that configs related to the correctness of the Root/Metadata tables
+ // are initialized correctly
+ @Test
+ public void testSystemTablesInitialConfigCorrectness() throws Exception {
+ try (ClientContext client =
+ (ClientContext) Accumulo.newClient().from(getClientProps()).build()) {
+
+ // It is important here to use getTableProperties() and not
getConfiguration()
+ // because we want only the table properties and not a merged view
- var rootTableProps =
client.tableOperations().getTableProperties(RootTable.NAME);
- var metadataTableProps =
client.tableOperations().getTableProperties(MetadataTable.NAME);
++ var rootTableProps =
++
client.tableOperations().getTableProperties(AccumuloTable.ROOT.tableName());
++ var metadataTableProps =
++
client.tableOperations().getTableProperties(AccumuloTable.METADATA.tableName());
+
+ // Verify root table config
+ testCommonSystemTableConfig(rootTableProps);
+ assertEquals("root",
+
rootTableProps.get(Property.TABLE_COMPACTION_DISPATCHER_OPTS.getKey() +
"service"));
+
+ // Verify metadata table config
+ testCommonSystemTableConfig(metadataTableProps);
+ assertEquals("meta",
+
metadataTableProps.get(Property.TABLE_COMPACTION_DISPATCHER_OPTS.getKey() +
"service"));
+ }
+ }
+
+ private void testCommonSystemTableConfig(Map tableProps) {
+ // Verify properties all have a table. prefix
+ assertTrue(tableProps.keySet().stream().allMatch(key ->
key.startsWith("table.")));
+
+ // Verify properties are correctly set
+ assertEquals("5",
tableProps.get(Property.TABLE_FILE_REPLICATION.getKey()));
+ assertEquals("sync", tableProps.get(Property.TABLE_DURABILITY.getKey()));
+ assertEquals("false",
tableProps.get(Property.TABLE_FAILURES_IGNORE.getKey()));
+ assertEquals("",
tableProps.get(Property.TABLE_DEFAULT_SCANTIME_VISIBILITY.getKey()));
+ assertEquals("tablet,server",
tableProps.get(Property.TABLE_LOCALITY_GROUPS.getKey()));
+ assertEquals(
+ String.format("%s,%s",
MetadataSchema.TabletsSection.TabletColumnFamily.NAME,
+ MetadataSchema.TabletsSection.CurrentLocationColumnFamily.NAME),
+ tableProps.get(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() +
"tablet"));
+ assertEquals(
+ String.format("%s,%s,%s,%s",
MetadataSchema.TabletsSection.DataFileColumnFamily.NAME,
+ MetadataSchema.TabletsSection.LogColumnFamily.NAME,
+ MetadataSchema.TabletsSection.ServerColumnFamily.NAME,
+ MetadataSchema.TabletsSection.FutureLocationColumnFamily.NAME),
+ tableProps.get(Property.TABLE_LOCALITY_GROUP_PREFIX.getKey() +
"server"));
+ assertEquals("20," + MetadataBulkLoadFilter.class.getName(),
+ tableProps.get(Property.TABLE_ITERATOR_PREFIX.getKey() +
"majc.bulkLoadFilter"));
+ assertEquals(SimpleCompactionDispatcher.class.getName(),
+ tableProps.get(Property.TABLE_COMPACTION_DISPATCHER.getKey()));
+
+ // Verify VersioningIterator related properties are correct
+ var iterClass = "10," + VersioningIterator.class.getName();
+ var maxVersions = "1";
+ assertEquals(iterClass,
tableProps.get(Property.TABLE_ITERATOR_PREFIX.getKey() + "scan.vers"));
+ assertEquals(maxVersions,
+
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 981747433306236511fb652c4b7ebe5a461046ae
Merge: aed3919e46 48f150045b
Author: Daniel Roberts
AuthorDate: Fri Mar 1 00:51:19 2024 +
Merge branch '2.1'
.../main/java/org/apache/accumulo/core/Constants.java | 1 +
.../java/org/apache/accumulo/core/conf/Property.java | 18 --
.../spi/compaction/SimpleCompactionDispatcher.java | 10 +++---
.../accumulo/server/conf/CheckCompactionConfig.java| 4 +++-
.../test/compaction/BadCompactionServiceConfigIT.java | 4 +++-
5 files changed, 26 insertions(+), 11 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 6cb1207f40,1db7d9326d..e2afb7defb
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -637,25 -685,23 +639,29 @@@ public enum Property
PropertyType.JSON,
"See {% jlink -f
org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner %}.",
"2.1.0"),
+ @Deprecated(since = "3.1")
-
TSERV_COMPACTION_SERVICE_DEFAULT_PLANNER("tserver.compaction.major.service.default.planner",
+ TSERV_COMPACTION_SERVICE_DEFAULT_PLANNER(
+ "tserver.compaction.major.service." + DEFAULT_COMPACTION_SERVICE_NAME +
".planner",
DefaultCompactionPlanner.class.getName(), PropertyType.CLASSNAME,
"Planner for default compaction service.", "2.1.0"),
+ @Deprecated(since = "3.1")
-
TSERV_COMPACTION_SERVICE_DEFAULT_RATE_LIMIT("tserver.compaction.major.service.default.rate.limit",
- "0B", PropertyType.BYTES,
+ TSERV_COMPACTION_SERVICE_DEFAULT_RATE_LIMIT(
+ "tserver.compaction.major.service." + DEFAULT_COMPACTION_SERVICE_NAME +
".rate.limit", "0B",
+ PropertyType.BYTES,
"Maximum number of bytes to read or write per second over all major"
- + " compactions in this compaction service, or 0B for unlimited.",
+ + " compactions in this compaction service, or 0B for unlimited.
This property has"
+ + " been deprecated in anticipation of it being removed in a future
release that"
+ + " removes the rate limiting feature.",
"2.1.0"),
+ @Deprecated(since = "3.1")
TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN(
- "tserver.compaction.major.service.default.planner.opts.maxOpen", "10",
PropertyType.COUNT,
- "The maximum number of files a compaction will open.", "2.1.0"),
+ "tserver.compaction.major.service." + DEFAULT_COMPACTION_SERVICE_NAME
+ + ".planner.opts.maxOpen",
+ "10", PropertyType.COUNT, "The maximum number of files a compaction
will open.", "2.1.0"),
+ @Deprecated(since = "3.1")
TSERV_COMPACTION_SERVICE_DEFAULT_EXECUTORS(
- "tserver.compaction.major.service.default.planner.opts.executors",
+ "tserver.compaction.major.service." + DEFAULT_COMPACTION_SERVICE_NAME
+ + ".planner.opts.executors",
"[{'name':'small','type':'internal','maxSize':'32M','numThreads':2},{'name':'medium','type':'internal','maxSize':'128M','numThreads':2},{'name':'large','type':'internal','numThreads':2}]"
.replaceAll("'", "\""),
PropertyType.STRING,
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit ea5afb9d85e342834554504cdf541a43dcc79a84
Merge: 9ffc450b0a bdec3a7c3a
Author: Keith Turner
AuthorDate: Thu Feb 22 19:00:28 2024 -0500
Merge branch '2.1'
.../server/manager/state/MetaDataStateStore.java | 3 +-
.../manager/state/TabletStateChangeIterator.java | 3 +
.../accumulo/test/manager/SuspendedTabletsIT.java | 249 +
3 files changed, 157 insertions(+), 98 deletions(-)
diff --cc
test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java
index ec99094346,764bc1dfc9..8ebb75378b
---
a/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/manager/SuspendedTabletsIT.java
@@@ -333,6 -300,95 +303,93 @@@ public class SuspendedTabletsIT extend
throws Exception;
}
+ private class ShutdownTserverKiller implements TServerKiller {
+
+ @Override
+ public void eliminateTabletServers(ClientContext ctx, TabletLocations
locs, int count)
+ throws Exception {
-
+ Set tserverSet = new HashSet<>();
+ Set metadataServerSet = new HashSet<>();
+
- TabletLocator tl = TabletLocator.getLocator(ctx, MetadataTable.ID);
++ TabletLocator tl = TabletLocator.getLocator(ctx,
AccumuloTable.METADATA.tableId());
+ for (TabletLocationState tls : locs.locationStates.values()) {
+ if (tls.current != null) {
+ // add to set of all servers
+ tserverSet.add(tls.current.getServerInstance());
+
+ // get server that the current tablets metadata is on
+ TabletLocator.TabletLocation tab =
+ tl.locateTablet(ctx, tls.extent.toMetaRow(), false, false);
+ // add it to the set of servers with metadata
- metadataServerSet
- .add(new TServerInstance(tab.tablet_location,
Long.valueOf(tab.tablet_session, 16)));
++ metadataServerSet.add(new TServerInstance(tab.getTserverLocation(),
++ Long.valueOf(tab.getTserverSession(), 16)));
+ }
+ }
+
+ // remove servers with metadata on them from the list of servers to be
shutdown
+ assertEquals(1, metadataServerSet.size(), "Expecting a single tServer
in metadataServerSet");
+ tserverSet.removeAll(metadataServerSet);
+
+ assertEquals(TSERVERS - 1, tserverSet.size(),
+ "Expecting " + (TSERVERS - 1) + " tServers in shutdown-list");
+
+ List tserversList = new ArrayList<>(tserverSet);
- Collections.shuffle(tserversList, random);
++ Collections.shuffle(tserversList, RANDOM.get());
+
+ for (int i1 = 0; i1 < count; ++i1) {
+ final String tserverName = tserversList.get(i1).getHostPortSession();
+ ThriftClientTypes.MANAGER.executeVoid(ctx, client -> {
+ log.info("Sending shutdown command to {} via ManagerClientService",
tserverName);
+ client.shutdownTabletServer(null, ctx.rpcCreds(), tserverName,
false);
+ });
+ }
+
+ log.info("Waiting for tserver process{} to die", count == 1 ? "" :
"es");
+ for (int i2 = 0; i2 < 10; ++i2) {
+ List deadProcs = new ArrayList<>();
+ for (ProcessReference pr1 :
getCluster().getProcesses().get(ServerType.TABLET_SERVER)) {
+ Process p = pr1.getProcess();
+ if (!p.isAlive()) {
+ deadProcs.add(pr1);
+ }
+ }
+ for (ProcessReference pr2 : deadProcs) {
+ log.info("Process {} is dead, informing cluster control about
this", pr2.getProcess());
+
getCluster().getClusterControl().killProcess(ServerType.TABLET_SERVER, pr2);
+ --count;
+ }
+ if (count == 0) {
+ return;
+ } else {
+ Thread.sleep(SECONDS.toMillis(2));
+ }
+ }
+ throw new IllegalStateException("Tablet servers didn't die!");
-
+ }
+ }
+
+ private class CrashTserverKiller implements TServerKiller {
+
+ @Override
+ public void eliminateTabletServers(ClientContext ctx, TabletLocations
locs, int count)
+ throws Exception {
+ // Exclude the tablet server hosting the metadata table from the list
and only
+ // kill tablet servers that are not hosting the metadata table.
+ List procs =
getCluster().getProcesses().get(ServerType.TABLET_SERVER)
+ .stream().filter(p ->
!metadataTserverProcess.equals(p)).collect(Collectors.toList());
- Collections.shuffle(procs, random);
++ Collections.shuffle(procs, RANDOM.get());
+ assertEquals(TSERVERS - 1, procs.size(), "Not enough tservers exist");
+ assertTrue(procs.size() >= count, "Attempting to kill more tservers ("
+ count
+ + ") than exist in the cluster (" + procs.size() + ")");
+
+ for (int i = 0; i < count; ++i) {
+
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 9ffc450b0a0c15b2fe16a070b323250e82bca00d Merge: 9deb242ff6 1a40aee541 Author: Keith Turner AuthorDate: Thu Feb 22 16:11:00 2024 -0500 Merge branch '2.1' .../apache/accumulo/test/ScanConsistencyIT.java| 33 -- 1 file changed, 18 insertions(+), 15 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 9deb242ff63fd6dd2d0a9e72ac37ebd28cb7e84b Merge: dd7b749102 f2adaebba2 Author: Dave Marion AuthorDate: Thu Feb 22 17:50:39 2024 + Merge branch '2.1'
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit dd7b749102ad1718c1c0dc5cc46e0126c1a10023
Merge: e3d6204305 af50af77ce
Author: Dave Marion
AuthorDate: Thu Feb 22 17:28:33 2024 +
Merge branch '2.1'
.../manager/upgrade/UpgradeCoordinator.java| 37 +-
1 file changed, 36 insertions(+), 1 deletion(-)
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java
index f5fd84c0e1,d62e824e05..34b362fa2e
---
a/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgradeCoordinator.java
@@@ -190,25 -198,19 +203,28 @@@ public class UpgradeCoordinator
.submit(() -> {
try {
for (int v = currentVersion; v < AccumuloDataVersion.get();
v++) {
-log.info("Upgrading Root from data version {}", v);
+log.info("Upgrading Root - current version {} as step towards
target version {}", v,
+AccumuloDataVersion.get());
+var upgrader = upgraders.get(v);
+Objects.requireNonNull(upgrader,
+"upgrade root: failed to find root upgrader for version "
+ currentVersion);
upgraders.get(v).upgradeRoot(context);
}
-
setStatus(UpgradeStatus.UPGRADED_ROOT, eventCoordinator);
for (int v = currentVersion; v < AccumuloDataVersion.get();
v++) {
-log.info("Upgrading Metadata from data version {}", v);
+log.info(
+"Upgrading Metadata - current version {} as step towards
target version {}", v,
+AccumuloDataVersion.get());
+var upgrader = upgraders.get(v);
+Objects.requireNonNull(upgrader,
+"upgrade metadata: failed to find upgrader for version "
+ currentVersion);
upgraders.get(v).upgradeMetadata(context);
}
+ setStatus(UpgradeStatus.UPGRADED_METADATA, eventCoordinator);
+
+ log.info("Validating configuration properties.");
+ validateProperties(context);
log.info("Updating persistent data version.");
updateAccumuloVersion(context.getServerDirs(),
context.getVolumeManager(),
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 9f430a26956449133d13c1a35c367df7ce33dc7e
Merge: 978f8b9eca 3298d6db89
Author: Daniel Roberts
AuthorDate: Fri Feb 16 22:12:55 2024 +
Merge branch '2.1'
.../org/apache/accumulo/core/conf/Property.java | 10 ++
.../accumulo/coordinator/CompactionCoordinator.java | 5 ++---
.../org/apache/accumulo/compactor/Compactor.java| 21 ++---
3 files changed, 26 insertions(+), 10 deletions(-)
diff --cc
server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index 00956edbf1,1434d2a1f1..b81c30dee3
---
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@@ -579,15 -592,17 +584,17 @@@ public class Compactor extends Abstract
protected long getWaitTimeBetweenCompactionChecks() {
// get the total number of compactors assigned to this queue
int numCompactors = ExternalCompactionUtil.countCompactors(queueName,
getContext());
- // Aim for around 3 compactors checking in every second
- long sleepTime = numCompactors * 1000L / 3;
- // Ensure a compactor sleeps at least around a second
- sleepTime = Math.max(1000, sleepTime);
- // Ensure a compactor sleep not too much more than 5 mins
- sleepTime = Math.min(300_000L, sleepTime);
+ long minWait =
getConfiguration().getTimeInMillis(Property.COMPACTOR_MIN_JOB_WAIT_TIME);
+ // Aim for around 3 compactors checking in per min wait time.
+ long sleepTime = numCompactors * minWait / 3;
+ // Ensure a compactor waits at least the minimum time
+ sleepTime = Math.max(minWait, sleepTime);
+ // Ensure a sleeping compactor has a configurable max sleep time
+ sleepTime =
Math.min(getConfiguration().getTimeInMillis(Property.COMPACTOR_MAX_JOB_WAIT_TIME),
+ sleepTime);
// Add some random jitter to the sleep time, that averages out to sleep
time. This will spread
// compactors out evenly over time.
-sleepTime = (long) (.9 * sleepTime + sleepTime * .2 *
random.nextDouble());
+sleepTime = (long) (.9 * sleepTime + sleepTime * .2 *
RANDOM.get().nextDouble());
LOG.trace("Sleeping {}ms based on {} compactors", sleepTime,
numCompactors);
return sleepTime;
}
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 978f8b9ecaac2518ad3f2559512cf11d5fdd3901
Merge: d91178f44c f28b55dc25
Author: Keith Turner
AuthorDate: Tue Feb 13 17:12:35 2024 +
Merge branch '2.1'
.../accumulo/test/functional/CompactionIT.java | 20 +---
1 file changed, 13 insertions(+), 7 deletions(-)
diff --cc
test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
index 6df72f17d9,46aa609e2c..18fdff715d
--- a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
@@@ -81,9 -69,9 +81,10 @@@ import org.apache.accumulo.core.iterato
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iterators.user.AgeOffFilter;
import org.apache.accumulo.core.iterators.user.GrepIterator;
-import org.apache.accumulo.core.metadata.MetadataTable;
+import org.apache.accumulo.core.metadata.AccumuloTable;
+import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.schema.Ample;
+ import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
@@@ -985,9 -715,14 +986,14 @@@ public class CompactionIT extends Accum
}
}
- private int countFiles(AccumuloClient c) throws Exception {
+ /**
+* Counts the number of tablets and files in a table.
+*/
+ private int countFiles(AccumuloClient c, String tableName) throws Exception
{
+ var tableId = getCluster().getServerContext().getTableId(tableName);
-try (Scanner s = c.createScanner(MetadataTable.NAME,
Authorizations.EMPTY)) {
+try (Scanner s = c.createScanner(AccumuloTable.METADATA.tableName(),
Authorizations.EMPTY)) {
- s.fetchColumnFamily(new Text(TabletColumnFamily.NAME));
+ s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
+ TabletColumnFamily.PREV_ROW_COLUMN.fetch(s);
s.fetchColumnFamily(new Text(DataFileColumnFamily.NAME));
return Iterators.size(s.iterator());
}
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 171fdcd7f539da7b4ee0c011b5d546d3af6bde96
Merge: 113e495dbf 12919d094b
Author: Dave Marion
AuthorDate: Fri Feb 9 14:29:37 2024 +
Merge branch '2.1'
.../java/org/apache/accumulo/core/Constants.java | 2 -
.../org/apache/accumulo/core/conf/Property.java| 4 +-
.../MiniAccumuloClusterClasspathTest.java | 2 +-
.../minicluster/MiniAccumuloClusterTest.java | 4 +-
.../accumulo/tserver/BulkFailedCopyProcessor.java | 81 --
.../org/apache/accumulo/tserver/TabletServer.java | 20 +-
.../org/apache/accumulo/tserver/log/LogSorter.java | 25 ---
7 files changed, 19 insertions(+), 119 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/Constants.java
index 3c8da35361,00b1a2fd18..66ac9144a8
--- a/core/src/main/java/org/apache/accumulo/core/Constants.java
+++ b/core/src/main/java/org/apache/accumulo/core/Constants.java
@@@ -88,8 -88,9 +88,6 @@@ public class Constants
public static final String ZNEXT_FILE = "/next_file";
- // TODO: Remove when Property.TSERV_WORKQ_THREADS is removed
-- public static final String ZBULK_FAILED_COPYQ = "/bulk_failed_copyq";
--
public static final String ZHDFS_RESERVATIONS = "/hdfs_reservations";
public static final String ZRECOVERY = "/recovery";
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 56f856410d,1fa04490fb..9acce283cd
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -721,9 -811,10 +721,11 @@@ public enum Property
+ " that begin with 'table.file' can be used here. For example, to
set the compression"
+ " of the sorted recovery files to snappy use
'tserver.wal.sort.file.compress.type=snappy'.",
"2.1.0"),
+ @Deprecated(since = "2.1.3")
TSERV_WORKQ_THREADS("tserver.workq.threads", "2", PropertyType.COUNT,
"The number of threads for the distributed work queue. These threads
are"
- + " used for copying failed bulk import RFiles.",
- + " used for copying failed bulk import RFiles. This property will
be removed when bulk import V1 is removed.",
++ + " used for copying failed bulk import RFiles. Note that as of
version 3.1.0 this property"
++ + " is not used and will be removed in a future release.",
"1.4.2"),
TSERV_WAL_SYNC("tserver.wal.sync", "true", PropertyType.BOOLEAN,
"Use the SYNC_BLOCK create flag to sync WAL writes to disk. Prevents"
diff --cc
minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterClasspathTest.java
index 2ba1960701,d18ffdd4f0..2a5da748db
---
a/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterClasspathTest.java
+++
b/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterClasspathTest.java
@@@ -78,7 -80,7 +78,7 @@@ public class MiniAccumuloClusterClasspa
MiniAccumuloConfig config = new MiniAccumuloConfig(testDir,
ROOT_PASSWORD).setJDWPEnabled(true);
config.setZooKeeperPort(0);
HashMap site = new HashMap<>();
- site.put(Property.TSERV_WORKQ_THREADS.getKey(), "2");
-site.put(VFS_CONTEXT_CLASSPATH_PROPERTY.getKey() + "cx1",
jarFile.toURI().toString());
++site.put(Property.TSERV_WAL_MAX_SIZE.getKey(), "1G");
config.setSiteConfig(site);
accumulo = new MiniAccumuloCluster(config);
accumulo.start();
diff --cc
minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java
index 91f5b884c5,913904ef1f..d4a48de83a
---
a/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java
+++
b/minicluster/src/test/java/org/apache/accumulo/minicluster/MiniAccumuloClusterTest.java
@@@ -81,7 -79,7 +81,7 @@@ public class MiniAccumuloClusterTest ex
MiniAccumuloConfig config = new MiniAccumuloConfig(testDir,
ROOT_PASSWORD).setJDWPEnabled(true);
config.setZooKeeperPort(0);
HashMap site = new HashMap<>();
- site.put(Property.TSERV_WORKQ_THREADS.getKey(), "2");
-site.put(Property.TSERV_COMPACTION_WARN_TIME.getKey(), "5m");
++site.put(Property.TSERV_WAL_SORT_BUFFER_SIZE.getKey(), "15%");
config.setSiteConfig(site);
accumulo = new MiniAccumuloCluster(config);
accumulo.start();
@@@ -198,7 -194,7 +198,7 @@@
// ensure what user passed in is what comes back
assertEquals(0, accumulo.getConfig().getZooKeeperPort());
HashMap site = new HashMap<>();
- site.put(Property.TSERV_WORKQ_THREADS.getKey(), "2");
-site.put(Property.TSERV_COMPACTION_WARN_TIME.getKey(), "5m");
++site.put(Property.TSERV_WAL_SORT_BUFFER_SIZE.getKey(), "15%");
assertEquals(site, accumulo.getConfig().getSiteConfig());
}
diff --cc
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 113e495dbff884309c4213b9f5016c789850fe42 Merge: 1443a7edaf 896e6277e1 Author: Keith Turner AuthorDate: Thu Feb 8 13:56:27 2024 -0500 Merge branch '2.1' .../src/main/java/org/apache/accumulo/compactor/Compactor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit dfda8bbea49b04e440be50c204182df4f9e4f203 Merge: 8bb8fcea89 e97bcf2668 Author: Keith Turner AuthorDate: Wed Feb 7 22:00:50 2024 + Merge branch '2.1' .../src/main/java/org/apache/accumulo/tserver/ScanServer.java | 8 1 file changed, 4 insertions(+), 4 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 8bb8fcea890a378608b101cb015fc921b684fde9 Merge: 354be4fbaa f98c87cb57 Author: Daniel Roberts AuthorDate: Wed Feb 7 00:20:23 2024 + Merge branch '2.1' .../apache/accumulo/server/compaction/RetryableThriftCall.java | 9 - .../src/main/java/org/apache/accumulo/tserver/ScanServer.java| 2 +- 2 files changed, 9 insertions(+), 2 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 37782d558e81115a63bc85e9ada9e6ddf51d11a9 Merge: 5fb05efd26 cdd86f18f2 Author: Daniel Roberts AuthorDate: Fri Feb 2 04:33:24 2024 + Merge branch '2.1'
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 5fb05efd26d0c15f633dbd4e0d9ce8daa16282a5
Merge: 1db892b00d 8bf0970c7f
Author: Keith Turner
AuthorDate: Thu Feb 1 17:16:50 2024 -0500
Merge branch '2.1'
.../tserver/compactions/CompactionManager.java | 8 +-
.../tserver/compactions/CompactionService.java | 1 +
.../accumulo/test/functional/CompactionIT.java | 88 ++
3 files changed, 93 insertions(+), 4 deletions(-)
diff --cc
test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
index 7b633e338d,93c8c1713e..6df72f17d9
--- a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
@@@ -90,8 -77,9 +90,10 @@@ import org.apache.accumulo.core.metadat
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.security.Authorizations;
+ import org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner;
+ import org.apache.accumulo.core.spi.compaction.SimpleCompactionDispatcher;
import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.minicluster.ServerType;
import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
import org.apache.accumulo.test.VerifyIngest;
import org.apache.accumulo.test.VerifyIngest.VerifyParams;
@@@ -705,200 -628,94 +707,286 @@@ public class CompactionIT extends Accum
}
}
+ @Test
+ public void testSelectNoFiles() throws Exception {
+
+// Test a compaction selector that selects no files. In this case there
is no work to,
+// so we want to ensure it does not hang
+
+try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+
+ writeFlush(c, tableName, "a");
+ writeFlush(c, tableName, "b");
+
+ CompactionConfig config = new CompactionConfig()
+ .setSelector(new
PluginConfig(EmptyCompactionSelector.class.getName(), Map.of()))
+ .setWait(true);
+ c.tableOperations().compact(tableName, config);
+
+ assertEquals(Set.of("a", "b"), getRows(c, tableName));
+}
+
+ }
+
+ @Test
+ public void testConcurrent() throws Exception {
+// two compactions without iterators or strategy should be able to run
concurrently
+
+try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
+
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+
+ // write random data because its very unlikely it will compress
+ writeRandomValue(c, tableName, 1 << 16);
+ writeRandomValue(c, tableName, 1 << 16);
+
+ c.tableOperations().compact(tableName, new
CompactionConfig().setWait(false));
+ c.tableOperations().compact(tableName, new
CompactionConfig().setWait(true));
+
+ assertEquals(1, FunctionalTestUtils.countRFiles(c, tableName));
+
+ writeRandomValue(c, tableName, 1 << 16);
+
+ IteratorSetting iterConfig = new IteratorSetting(30,
SlowIterator.class);
+ SlowIterator.setSleepTime(iterConfig, 1000);
+
+ long t1 = System.currentTimeMillis();
+ c.tableOperations().compact(tableName,
+ new
CompactionConfig().setWait(false).setIterators(java.util.Arrays.asList(iterConfig)));
+ try {
+// this compaction should fail because previous one set iterators
+c.tableOperations().compact(tableName, new
CompactionConfig().setWait(true));
+if (System.currentTimeMillis() - t1 < 2000) {
+ fail("Expected compaction to fail because another concurrent
compaction set iterators");
+}
+ } catch (AccumuloException e) {}
+}
+ }
+
+ @Test
+ public void testGetSelectedFilesForCompaction() throws Exception {
+
+// Tests CompactionConfigurer.InputParameters.getSelectedFiles()
+
+String tableName = this.getUniqueNames(1)[0];
+// Disable GC so intermediate compaction files are not deleted
+
getCluster().getClusterControl().stopAllServers(ServerType.GARBAGE_COLLECTOR);
+
+try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ Map props = new HashMap<>();
+ props.put(Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "none");
+ // This is done to avoid system compactions - we want to do all the
compactions ourselves
+ props.put("table.compaction.dispatcher.opts.service.system",
"nonexitant");
+ NewTableConfiguration ntc = new
NewTableConfiguration().setProperties(props);
+ client.tableOperations().create(tableName, ntc);
+
+ // The following will create 4 small and 4 large
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit bce2ef2bfc8b3a80dee5cf3e7313fb99b23bb7d1 Merge: b85cafb61c 1531fe5d7e Author: Christopher L. Shannon AuthorDate: Sat Jan 27 10:38:04 2024 -0500 Merge branch '2.1' .../security/delegation/AuthenticationTokenSecretManagerTest.java | 6 ++ 1 file changed, 6 insertions(+) diff --cc server/base/src/test/java/org/apache/accumulo/server/security/delegation/AuthenticationTokenSecretManagerTest.java index 8a7b6ea7bf,00d9e6cf03..7198e16883 --- a/server/base/src/test/java/org/apache/accumulo/server/security/delegation/AuthenticationTokenSecretManagerTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/security/delegation/AuthenticationTokenSecretManagerTest.java @@@ -207,9 -207,13 +207,15 @@@ public class AuthenticationTokenSecretM // The passwords line up against multiple calls with the same ID assertArrayEquals(password, secretManager.retrievePassword(id)); + // Sleep 50 ms to make sure we generate another token for the test + // System.currentTimeMillis() is used as part of the token generation and if + // the test runs fast enough it can return the same value that was used + // when generating the first token and the test will fail + Thread.sleep(50); + // Make a second token for the same user +// Briefly sleep to guarantee token is unique, since the token is based on the time +Thread.sleep(100); Entry,AuthenticationTokenIdentifier> pair2 = secretManager.generateToken(principal, cfg); Token token2 = pair2.getKey();
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit b85cafb61c5d3036a40a8e0c9ca9da3f02302d5d
Merge: 97f9cf80cb 5401dedda0
Author: Keith Turner
AuthorDate: Fri Jan 26 15:26:23 2024 -0500
Merge branch '2.1'
.../accumulo/core/iterators/user/RowFilter.java| 7 +-
.../core/iterators/user/RowFilterTest.java | 89 ++
2 files changed, 95 insertions(+), 1 deletion(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/iterators/user/RowFilter.java
index a6849e448b,54d8b0978d..ac93ab1455
--- a/core/src/main/java/org/apache/accumulo/core/iterators/user/RowFilter.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/RowFilter.java
@@@ -19,6 -19,6 +19,7 @@@
package org.apache.accumulo.core.iterators.user;
import java.io.IOException;
++import java.io.UncheckedIOException;
import java.util.Collection;
import java.util.Map;
@@@ -155,10 -157,10 +158,12 @@@ public abstract class RowFilter extend
RowFilter newInstance;
try {
newInstance = getClass().getDeclaredConstructor().newInstance();
+ newInstance.init(getSource().deepCopy(env), options, env);
-} catch (Exception e) {
- throw new RuntimeException(e);
+} catch (ReflectiveOperationException e) {
+ throw new IllegalStateException(e);
++} catch (IOException e) {
++ throw new UncheckedIOException(e);
}
- newInstance.setSource(getSource().deepCopy(env));
newInstance.decisionIterator = new RowIterator(getSource().deepCopy(env));
return newInstance;
}
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 5910742acacc41f199e3423597f077e236ee7eea Merge: 731bbf75e7 ac2886c34e Author: Christopher Tubbs AuthorDate: Tue Jan 23 17:36:00 2024 -0500 Merge branch '2.1' shell/src/main/java/org/apache/accumulo/shell/ShellOptionsJC.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 5922d90b95bcd611ed77515f4097af2473b0519b Merge: bed5e3628d c0d548aae3 Author: Daniel Roberts AuthorDate: Mon Jan 22 16:44:42 2024 + Merge branch '2.1' assemble/src/main/resources/LICENSE| 42 +++--- .../accumulo/server/compaction/FileCompactor.java | 16 +++-- .../coordinator/CompactionCoordinator.java | 3 +- .../accumulo/coordinator/CompactionFinalizer.java | 3 +- .../coordinator/DeadCompactionDetector.java| 4 +-- .../org/apache/accumulo/compactor/Compactor.java | 11 -- .../accumulo/tserver/tablet/CompactableImpl.java | 5 +-- .../compaction/ExternalDoNothingCompactor.java | 5 ++- 8 files changed, 55 insertions(+), 34 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit bed5e3628d9777d2e84587cced2fe9d9989accba Merge: 9a5a24fec2 0071ee070f Author: Daniel Roberts AuthorDate: Mon Jan 22 14:48:34 2024 + Merge branch '2.1' assemble/conf/accumulo-env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 2c4ffc28593978f30c51de3263c437ac6deaa8c4 Merge: 810d4b10e6 73107e263d Author: Keith Turner AuthorDate: Mon Jan 8 11:35:42 2024 -0500 Merge branch '2.1' .../src/main/java/org/apache/accumulo/test/functional/CompactionIT.java | 2 ++ 1 file changed, 2 insertions(+)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. domgarguilo pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 810d4b10e6381aa2ea73ca46b2e59f4f6a7334de Merge: 6dc1bcfa55 59b1c170a3 Author: Dom G AuthorDate: Fri Jan 5 11:15:28 2024 -0500 Merge branch '2.1' .../core/client/admin/NamespaceOperations.java | 18 +++--- .../core/client/admin/TableOperations.java | 28 -- .../accumulo/manager/FateServiceHandler.java | 7 +++--- .../tableOps/tableImport/FinishImportTable.java| 5 ++-- .../manager/tableOps/tableImport/ImportTable.java | 4 ++-- .../tableOps/tableImport/ImportedTableInfo.java| 2 +- .../org/apache/accumulo/test/ImportExportIT.java | 2 ++ 7 files changed, 35 insertions(+), 31 deletions(-) diff --cc test/src/main/java/org/apache/accumulo/test/ImportExportIT.java index 923ed5d8ab,49f01c1b36..0e1242dcc1 --- a/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java @@@ -46,11 -45,10 +46,12 @@@ import org.apache.accumulo.core.client. import org.apache.accumulo.core.client.admin.ImportConfiguration; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; + import org.apache.accumulo.core.manager.state.tables.TableState; import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 6dc1bcfa559be2fe8cab1b8e28552b44f7008957
Merge: 5583129435 08628c016b
Author: Keith Turner
AuthorDate: Thu Jan 4 19:55:52 2024 -0500
Merge branch '2.1'
.../org/apache/accumulo/core/conf/Property.java| 8 +-
.../spi/compaction/DefaultCompactionPlanner.java | 106 ++-
.../compaction/DefaultCompactionPlannerTest.java | 194 -
3 files changed, 300 insertions(+), 8 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
index 8f7969519a,aae0591567..eba25df062
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
@@@ -115,31 -105,26 +116,46 @@@ import edu.umd.cs.findbugs.annotations.
*
* Note that the use of 'external' requires that the CompactionCoordinator
and at least one
* Compactor for Queue1 is running.
- * {@code tserver.compaction.major.service..opts.maxOpen} This
determines the maximum
- * number of files that will be included in a single compaction.
+ * {@code compaction.service..opts.maxOpen} This determines the
maximum number of files
+ * that will be included in a single compaction.
+ * {@code compaction.service..opts.queues} This is a json array
of queue objects which
+ * have the following fields:
+ *
+ * Default Compaction Planner Queue options
+ *
+ * Field Name
+ * Description
+ *
+ *
+ * name
+ * name or alias of the queue (required)
+ *
+ *
+ * maxSize
+ * threshold sum of the input files (required for all but one of the
configs)
+ *
+ *
+ *
+ * This 'queues' object is used for defining external compaction queues
without needing to use the
+ * thread-based 'executors' property.
*
*
+ *
+ * Starting with Accumulo 2.1.3, this plugin will use the table config option
+ * {@code "table.file.max"}. When the following four conditions are met, then
this plugin will try
+ * to find a lower compaction ratio that will result in a compaction:
+ *
+ * When a tablet has no compactions running
+ * Its number of files exceeds table.file.max
+ * System compactions are not finding anything to compact
+ * No files are selected for user compaction
+ *
+ * For example, given a tablet with 20 files, and table.file.max is 15 and no
compactions are
+ * planned. If the compaction ratio is set to 3, then this plugin will find
the largest compaction
+ * ratio less than 3 that results in a compaction.
+ *
+ *
- * @since 2.1.0
+ * @since 3.1.0
* @see org.apache.accumulo.core.spi.compaction
*/
@@@ -280,107 -226,118 +296,119 @@@ public class DefaultCompactionPlanner i
determineMaxFilesToCompact(params);
}
- @SuppressWarnings("removal")
+ @SuppressWarnings("deprecation")
private void determineMaxFilesToCompact(InitParameters params) {
-String fqo = params.getFullyQualifiedOption("maxOpen");
-if (!params.getServiceEnvironment().getConfiguration().isSet(fqo)
-&& params.getServiceEnvironment().getConfiguration()
-.isSet(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey())) {
- log.warn("The property " + Property.TSERV_MAJC_THREAD_MAXOPEN.getKey()
- + " was set, it is deprecated. Set the " + fqo + " option
instead.");
- this.maxFilesToCompact =
Integer.parseInt(params.getServiceEnvironment().getConfiguration()
- .get(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey()));
-} else {
- this.maxFilesToCompact =
Integer.parseInt(params.getOptions().getOrDefault("maxOpen",
-
Property.TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN.getDefaultValue()));
+
+String maxOpen = params.getOptions().get("maxOpen");
+if (maxOpen == null) {
+ maxOpen =
Property.TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN.getDefaultValue();
+ log.trace("default maxOpen not set, defaulting to {}", maxOpen);
}
+this.maxFilesToCompact = Integer.parseInt(maxOpen);
}
- @Override
- public CompactionPlan makePlan(PlanningParameters params) {
-try {
+ private void validateConfig(JsonElement json, List fields, String
className) {
- if (params.getCandidates().isEmpty()) {
-return params.createPlanBuilder().build();
- }
+JsonObject jsonObject = GSON.get().fromJson(json, JsonObject.class);
- Set filesCopy = new HashSet<>(params.getCandidates());
+List objectProperties = new ArrayList<>(jsonObject.keySet());
+HashSet classFieldNames = new HashSet<>(fields);
+
+if (!classFieldNames.containsAll(objectProperties)) {
+ objectProperties.removeAll(classFieldNames);
+ throw new JsonParseException(
+ "Invalid fields: " + objectProperties + " provided for class: " +
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit dc2800691e3b4e269d8f86d6b9f99861c4d9257d
Merge: 3792630c86 319ec30b7f
Author: Daniel Roberts
AuthorDate: Thu Jan 4 01:53:51 2024 +
Merge branch '2.1'
.../apache/accumulo/test/functional/MergeIT.java | 60 ++
1 file changed, 60 insertions(+)
diff --cc test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
index 4d265a1120,43590e8030..af02e42f51
--- a/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/MergeIT.java
@@@ -18,52 -18,46 +18,63 @@@
*/
package org.apache.accumulo.test.functional;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static
org.apache.accumulo.test.util.FileMetadataUtil.printAndVerifyFileMetadata;
+import static
org.apache.accumulo.test.util.FileMetadataUtil.verifyMergedMarkerCleared;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+ import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
import java.time.Duration;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Map.Entry;
+ import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+ import java.util.UUID;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.BatchWriter;
import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.admin.CompactionConfig;
import org.apache.accumulo.core.client.admin.NewTableConfiguration;
import org.apache.accumulo.core.client.admin.TimeType;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
+ import org.apache.accumulo.core.dataImpl.KeyExtent;
++import org.apache.accumulo.core.metadata.ReferencedTabletFile;
import org.apache.accumulo.core.metadata.StoredTabletFile;
-import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.metadata.schema.DataFileValue;
+ import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
+ import org.apache.accumulo.core.metadata.schema.ExternalCompactionMetadata;
import org.apache.accumulo.core.security.Authorizations;
+ import org.apache.accumulo.core.spi.compaction.CompactionExecutorId;
+ import org.apache.accumulo.core.spi.compaction.CompactionKind;
+import org.apache.accumulo.core.util.FastFormat;
import org.apache.accumulo.core.util.Merge;
+ import org.apache.accumulo.core.util.compaction.CompactionExecutorIdImpl;
import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.TestIngest.IngestParams;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.accumulo.test.VerifyIngest.VerifyParams;
+ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class MergeIT extends AccumuloClusterHarness {
@@@ -568,4 -230,52 +579,53 @@@
}
}
}
+
+ // Test that merge handles metadata from compactions
+ @Test
+ public void testCompactionMetadata() throws Exception {
+ try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+
+ var split = new Text("m");
+ c.tableOperations().addSplits(tableName, new TreeSet<>(List.of(split)));
+
+ TableId tableId = getServerContext().getTableId(tableName);
+
+ // add metadata from compactions to tablets prior to merge
+ try (var tabletsMutator =
getServerContext().getAmple().mutateTablets()) {
+ for (var extent : List.of(new KeyExtent(tableId, split, null),
+ new KeyExtent(tableId, null, split))) {
+ var tablet = tabletsMutator.mutateTablet(extent);
+ ExternalCompactionId ecid =
ExternalCompactionId.generate(UUID.randomUUID());
+
- TabletFile tmpFile = new TabletFile(new
Path("file:///accumulo/tables/t-0/b-0/c1.rf"));
++ ReferencedTabletFile tmpFile =
++ ReferencedTabletFile.of(new
Path("file:///accumulo/tables/t-0/b-0/c1.rf"));
+ CompactionExecutorId
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 3792630c867b2affb4f16efc9f537a04ca66cf3e
Merge: b8869fb72e d973179452
Author: Daniel Roberts
AuthorDate: Thu Dec 28 04:08:36 2023 +
Merge branch '2.1'
.../accumulo/server/metadata/ServerAmpleImpl.java | 18 ++
.../accumulo/test/functional/GarbageCollectorIT.java | 18 +++---
2 files changed, 29 insertions(+), 7 deletions(-)
diff --cc
server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
index 0e4fa2c478,ef02f22b4d..7aa6ecb6e1
---
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
@@@ -227,13 -226,23 +227,23 @@@ public class ServerAmpleImpl extends Am
}
try (BatchWriter writer = context.createBatchWriter(level.metaTable())) {
- for (GcCandidate candidate : candidates) {
- Mutation m = new
Mutation(DeletesSection.encodeRow(candidate.getPath()));
- m.putDelete(EMPTY_TEXT, EMPTY_TEXT, candidate.getUid());
- writer.addMutation(m);
+ if (type == GcCandidateType.VALID) {
+ for (GcCandidate candidate : candidates) {
+ Mutation m = new
Mutation(DeletesSection.encodeRow(candidate.getPath()));
+ // Removes all versions of the candidate to avoid reprocessing
deleted file entries
+ m.putDelete(EMPTY_TEXT, EMPTY_TEXT);
+ writer.addMutation(m);
+ }
+ } else {
+ for (GcCandidate candidate : candidates) {
+ Mutation m = new
Mutation(DeletesSection.encodeRow(candidate.getPath()));
+ // Removes this and older versions while allowing newer candidate
versions to persist
+ m.putDelete(EMPTY_TEXT, EMPTY_TEXT, candidate.getUid());
+ writer.addMutation(m);
+ }
}
} catch (MutationsRejectedException | TableNotFoundException e) {
- throw new RuntimeException(e);
+ throw new IllegalStateException(e);
}
}
diff --cc
test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
index 82c657e1a5,1776bc4c03..01ca7b8f5d
---
a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java
@@@ -497,11 -510,10 +506,11 @@@ public class GarbageCollectorIT extend
GcCandidate deleteCandidate = candidates.get(0);
assertNotNull(deleteCandidate);
-ample.putGcCandidates(tableId, List.of(new
StoredTabletFile(deleteCandidate.getPath(;
+ample.putGcCandidates(tableId,
+List.of(StoredTabletFile.of(new Path(deleteCandidate.getPath();
log.debug("Deleting Candidate {}", deleteCandidate);
- ample.deleteGcCandidates(datalevel, List.of(deleteCandidate),
Ample.GcCandidateType.INUSE);
+ ample.deleteGcCandidates(datalevel, List.of(deleteCandidate), type);
candidate = ample.getGcCandidates(datalevel);
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit b8869fb72e2aa58736250c52f3366f196ac86c79
Merge: 829634653b fdd73fb8b2
Author: Daniel Roberts
AuthorDate: Wed Dec 27 20:30:47 2023 +
Merge branch '2.1'
.../spi/compaction/DefaultCompactionPlanner.java | 6 +-
.../compaction/CompactionPlannerInitParams.java| 2 +-
.../compaction/DefaultCompactionPlannerTest.java | 343 ++---
3 files changed, 106 insertions(+), 245 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
index b73544143e,9385806831..8f7969519a
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
@@@ -279,106 -210,105 +280,107 @@@ public class DefaultCompactionPlanner i
determineMaxFilesToCompact(params);
}
- @SuppressWarnings("removal")
++ @SuppressWarnings("deprecation")
private void determineMaxFilesToCompact(InitParameters params) {
-String fqo = params.getFullyQualifiedOption("maxOpen");
-if (!params.getServiceEnvironment().getConfiguration().isSet(fqo)
-&& params.getServiceEnvironment().getConfiguration()
-.isSet(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey())) {
- log.warn("The property " + Property.TSERV_MAJC_THREAD_MAXOPEN.getKey()
- + " was set, it is deprecated. Set the " + fqo + " option
instead.");
- this.maxFilesToCompact =
Integer.parseInt(params.getServiceEnvironment().getConfiguration()
- .get(Property.TSERV_MAJC_THREAD_MAXOPEN.getKey()));
-} else {
- this.maxFilesToCompact =
Integer.parseInt(params.getOptions().getOrDefault("maxOpen",
-
Property.TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN.getDefaultValue()));
+
+String maxOpen = params.getOptions().get("maxOpen");
+if (maxOpen == null) {
- maxOpen = "10";
- log.trace("default maxOpen not set, defaulting to 10");
++ maxOpen =
Property.TSERV_COMPACTION_SERVICE_DEFAULT_MAX_OPEN.getDefaultValue();
++ log.trace("default maxOpen not set, defaulting to {}", maxOpen);
}
+this.maxFilesToCompact = Integer.parseInt(maxOpen);
}
- @Override
- public CompactionPlan makePlan(PlanningParameters params) {
-try {
+ private void validateConfig(JsonElement json, List fields, String
className) {
- if (params.getCandidates().isEmpty()) {
-return params.createPlanBuilder().build();
- }
+JsonObject jsonObject = GSON.get().fromJson(json, JsonObject.class);
+
+List objectProperties = new ArrayList<>(jsonObject.keySet());
+HashSet classFieldNames = new HashSet<>(fields);
+
+if (!classFieldNames.containsAll(objectProperties)) {
+ objectProperties.removeAll(classFieldNames);
+ throw new JsonParseException(
+ "Invalid fields: " + objectProperties + " provided for class: " +
className);
+}
+ }
- Set filesCopy = new HashSet<>(params.getCandidates());
+ @Override
+ public CompactionPlan makePlan(PlanningParameters params) {
+if (params.getCandidates().isEmpty()) {
+ return params.createPlanBuilder().build();
+}
- long maxSizeToCompact = getMaxSizeToCompact(params.getKind());
+Set filesCopy = new HashSet<>(params.getCandidates());
- Collection group;
- if (params.getRunningCompactions().isEmpty()) {
-group = findDataFilesToCompact(filesCopy, params.getRatio(),
maxFilesToCompact,
-maxSizeToCompact);
+long maxSizeToCompact = getMaxSizeToCompact(params.getKind());
-if (!group.isEmpty() && group.size() < params.getCandidates().size()
-&& params.getCandidates().size() <= maxFilesToCompact
-&& (params.getKind() == CompactionKind.USER
-|| params.getKind() == CompactionKind.SELECTOR)) {
- // USER and SELECTOR compactions must eventually compact all files.
When a subset of files
- // that meets the compaction ratio is selected, look ahead and see
if the next compaction
- // would also meet the compaction ratio. If not then compact
everything to avoid doing
- // more than logarithmic work across multiple comapctions.
+Collection group;
+if (params.getRunningCompactions().isEmpty()) {
+ group =
+ findDataFilesToCompact(filesCopy, params.getRatio(),
maxFilesToCompact, maxSizeToCompact);
- filesCopy.removeAll(group);
- filesCopy.add(getExpected(group, 0));
+ if (!group.isEmpty() && group.size() < params.getCandidates().size()
+ && params.getCandidates().size() <= maxFilesToCompact
+ && (params.getKind() == CompactionKind.USER
+ || params.getKind() ==
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit a6f22a32cedc4a50d75825813c1eef9ad9a0b0a2 Merge: 37117b2cb5 c7df850d83 Author: Christopher Tubbs AuthorDate: Thu Dec 21 02:13:19 2023 -0500 Merge branch '2.1' .github/ISSUE_TEMPLATE/post_vote_checklist.md | 5 +- .github/workflows/maven-full-its.yaml | 18 ++-- .github/workflows/maven-on-demand.yaml | 8 +- .github/workflows/maven.yaml | 21 +++-- .github/workflows/scripts.yaml | 6 +- assemble/pom.xml | 5 -- core/pom.xml | 4 - .../org/apache/accumulo/core/trace/TraceUtil.java | 8 +- pom.xml| 99 -- .../accumulo/server/util/RestoreZookeeper.java | 7 +- server/monitor/pom.xml | 6 ++ 11 files changed, 78 insertions(+), 109 deletions(-) diff --cc pom.xml index e49fd960d5,b982196bff..798c876af1 --- a/pom.xml +++ b/pom.xml @@@ -133,10 -134,9 +133,9 @@@ true .+-SNAPSHOT,(?i).*(alpha|beta)[0-9.-]*,(?i).*[.-](m|rc)[0-9]+ - 11 - 3.6.0 + 17 - 2023-08-14T08:01:47Z + 2023-08-14T08:11:10Z true source-release-tar @@@ -151,11 -151,11 +150,10 @@@ 5.5.0 2.23.0 3.3.6 - 3.6.0 - 1.27.0 - 2.0.7 + 1.33.0 -2.0.9 + 2.0.9 0.17.0 - 3.8.3 + 3.9.1 @@@ -295,8 -288,13 +286,8 @@@ com.google.guava guava - 32.0.1-jre + 33.0.0-jre - -com.google.protobuf -protobuf-java -3.22.0 - com.lmax disruptor @@@ -945,6 -985,10 +931,8 @@@ org.apache.logging.log4j:log4j-1.2-api:jar:* org.apache.logging.log4j:log4j-slf4j2-impl:jar:* org.apache.logging.log4j:log4j-web:jar:* + + biz.aQute.bnd:biz.aQute.bnd.annotation:jar:* - -org.junit.vintage:junit-vintage-engine:jar:* org.junit.jupiter:junit-jupiter-engine:jar:*
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 170f1b684bf31e36525447146b6a576b36bdeea8 Merge: f9897862dd 69381b494d Author: Keith Turner AuthorDate: Wed Dec 20 16:49:02 2023 -0500 Merge branch '2.1' .../core/spi/compaction/DefaultCompactionPlanner.java | 13 + .../accumulo/tserver/compactions/CompactionService.java | 9 + 2 files changed, 14 insertions(+), 8 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 436a6835de165715c930f9d6cebe1cad76d9d578
Merge: 3061ff028d 8dfb4eb713
Author: Christopher Tubbs
AuthorDate: Mon Dec 18 19:02:37 2023 -0500
Merge branch '2.1'
core/pom.xml | 4 ++--
.../java/org/apache/accumulo/core/conf/ClientConfigGenerate.java | 8
.../java/org/apache/accumulo/core/conf/ConfigurationDocGen.java | 4 ++--
3 files changed, 8 insertions(+), 8 deletions(-)
diff --cc core/pom.xml
index e1a83b0228,7869dddb38..2ba4af4861
--- a/core/pom.xml
+++ b/core/pom.xml
@@@ -355,7 -355,7 +355,7 @@@
test
--generate-markdown
--
${project.build.directory}/generated-docs/server-properties.md
++
${project.build.directory}/generated-docs/server-properties3.md
@@@ -370,7 -370,7 +370,7 @@@
test
--generate-markdown
--
${project.build.directory}/generated-docs/client-properties.md
++
${project.build.directory}/generated-docs/client-properties3.md
diff --cc
core/src/main/java/org/apache/accumulo/core/conf/ClientConfigGenerate.java
index d4d01d0786,0cd21758b1..f95688a4ed
--- a/core/src/main/java/org/apache/accumulo/core/conf/ClientConfigGenerate.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ClientConfigGenerate.java
@@@ -29,8 -29,8 +29,8 @@@ import java.util.TreeMap
import com.google.common.collect.Sets;
/**
-- * Generates client-properties.md for documentation on Accumulo website and
-- * accumulo-client.properties for Accumulo distribution tarball
++ * Generates client properties documentation on Accumulo website and
accumulo-client.properties for
++ * Accumulo distribution tarball
*/
public class ClientConfigGenerate {
@@@ -86,9 -86,9 +86,9 @@@
@Override
void pageHeader() {
doc.println("---");
- doc.println("title: Client Properties");
- doc.println("title: Client Properties (2.x)");
++ doc.println("title: Client Properties (3.x)");
doc.println("category: configuration");
-- doc.println("order: 3");
++ doc.println("order: 4");
doc.println("---\n");
doc.println("");
diff --cc
core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
index 163b150eab,51a6e24732..dc5320e93e
--- a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationDocGen.java
@@@ -62,9 -62,9 +62,9 @@@ public class ConfigurationDocGen
void pageHeader() {
doc.println("---");
- doc.println("title: Server Properties");
-doc.println("title: Server Properties (2.x)");
++doc.println("title: Server Properties (3.x)");
doc.println("category: configuration");
- doc.println("order: 4");
-doc.println("order: 5");
++doc.println("order: 6");
doc.println("---\n");
doc.println("\n");
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 593711095aba3847aa5e1b773a5cdcde0c4bacfc Merge: 4d38127e5a 5b46bd8f97 Author: Daniel Roberts AuthorDate: Wed Dec 13 19:58:32 2023 + Merge branch '2.1' .../org/apache/accumulo/test/fate/zookeeper/FateIT.java | 13 ++--- 1 file changed, 10 insertions(+), 3 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 4d38127e5a3f10e404e90323c49cf146f4c70007
Merge: 1aa81d81b0 e62ff01808
Author: Daniel Roberts
AuthorDate: Wed Dec 13 18:34:15 2023 +
Merge branch '2.1'
.../org/apache/accumulo/tserver/tablet/Tablet.java | 45 ++
1 file changed, 30 insertions(+), 15 deletions(-)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 7c741729df,6a646c4f57..65db951634
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@@ -1733,17 -1750,40 +1733,40 @@@ public class Tablet extends TabletBase
throw new IOException("tablet " + extent + " is closed");
}
- // TODO check seems unneeded now - ACCUMULO-1291
- long lockWait = System.currentTimeMillis() - now;
- if (lockWait
- >
getTabletServer().getConfiguration().getTimeInMillis(Property.GENERAL_RPC_TIMEOUT))
{
- throw new IOException(
- "Timeout waiting " + (lockWait / 1000.) + " seconds to get tablet
lock for " + extent);
+ long rpcTimeoutNanos = TimeUnit.MILLISECONDS.toNanos(
+ (long)
(getTabletServer().getConfiguration().getTimeInMillis(Property.GENERAL_RPC_TIMEOUT)
+ * 1.1));
+
+ // wait for any files that are bulk importing up to the RPC timeout
limit
+ while (!Collections.disjoint(bulkImporting, fileMap.keySet())) {
+ try {
+ wait(1_000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+
+ long lockWait = System.nanoTime() - now;
+ if (lockWait > rpcTimeoutNanos) {
+ throw new IOException("Timeout waiting " +
TimeUnit.NANOSECONDS.toSeconds(lockWait)
+ + " seconds to get tablet lock for " + extent + " " + tid);
+ }
+ }
+
+ // need to check this again because when wait is called above the lock
is released.
+ if (isClosed()) {
+ throw new IOException("tablet " + extent + " is closed");
+ }
+
+ long lockWait = System.nanoTime() - now;
+ if (lockWait > rpcTimeoutNanos) {
+ throw new IOException("Timeout waiting " +
TimeUnit.NANOSECONDS.toSeconds(lockWait)
+ + " seconds to get tablet lock for " + extent + " " + tid);
}
- List alreadyImported = bulkImported.get(tid);
+ List alreadyImported = bulkImported.get(tid);
if (alreadyImported != null) {
-for (TabletFile entry : alreadyImported) {
+for (ReferencedTabletFile entry : alreadyImported) {
if (fileMap.remove(entry) != null) {
log.trace("Ignoring import of bulk file already imported: {}",
entry);
}
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 1aa81d81b0ff0da4dcd9ed4e33e599d701297c01
Merge: 6cfa1b2ea7 01f48adf38
Author: Daniel Roberts
AuthorDate: Wed Dec 13 15:35:48 2023 +
Merge branch '2.1'
.../core/client/MutationsRejectedException.java| 11 ++-
.../server/constraints/MetadataConstraints.java| 20 +
.../constraints/MetadataConstraintsTest.java | 94 --
3 files changed, 24 insertions(+), 101 deletions(-)
diff --cc
server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
index aa9772a159,ced97d0a68..617bf20e86
---
a/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/constraints/MetadataConstraints.java
@@@ -390,9 -327,7 +374,9 @@@ public class MetadataConstraints implem
case 7:
return "Lock not held in zookeeper by writer";
case 8:
- return "Bulk load transaction no longer running";
+ return "Bulk load mutation contains either inconsistent files or
multiple fateTX ids";
+ case 9:
+return "Invalid data file metadata format";
}
return null;
}
diff --cc
server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
index 38f6d27ad2,bd59fef1e7..135a2bce60
---
a/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
+++
b/server/base/src/test/java/org/apache/accumulo/server/constraints/MetadataConstraintsTest.java
@@@ -40,8 -33,6 +40,7 @@@ import org.apache.accumulo.core.metadat
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily;
import org.apache.accumulo.server.ServerContext;
- import org.apache.accumulo.server.zookeeper.TransactionWatcher.Arbitrator;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.easymock.EasyMock;
import org.junit.jupiter.api.Test;
@@@ -158,376 -128,77 +136,332 @@@ public class MetadataConstraintsTest
Mutation m;
List violations;
- // inactive txid
- m = new Mutation(new Text("0;foo"));
- m.put(
- BulkFileColumnFamily.NAME, StoredTabletFile
- .of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile")).getMetadataText(),
- new Value("12345"));
- m.put(
- DataFileColumnFamily.NAME, StoredTabletFile
- .of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile")).getMetadataText(),
- new DataFileValue(1, 1).encodeAsValue());
- assertViolation(mc, m, (short) 8);
-
- // txid that throws exception
- m = new Mutation(new Text("0;foo"));
- m.put(
- BulkFileColumnFamily.NAME, StoredTabletFile
- .of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile")).getMetadataText(),
- new Value("9"));
- m.put(
- DataFileColumnFamily.NAME, StoredTabletFile
- .of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile")).getMetadataText(),
- new DataFileValue(1, 1).encodeAsValue());
- assertViolation(mc, m, (short) 8);
-
- // active txid w/ file
+ // loaded marker w/ file
m = new Mutation(new Text("0;foo"));
-m.put(BulkFileColumnFamily.NAME, new Text("/someFile"), new Value("5"));
-m.put(DataFileColumnFamily.NAME, new Text("/someFile"),
+m.put(
+BulkFileColumnFamily.NAME, StoredTabletFile
+.of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile")).getMetadataText(),
+new Value("5"));
+m.put(
+DataFileColumnFamily.NAME, StoredTabletFile
+.of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile")).getMetadataText(),
new DataFileValue(1, 1).encodeAsValue());
violations = mc.check(createEnv(), m);
assertNull(violations);
- // active txid w/o file
+ // loaded marker w/o file
m = new Mutation(new Text("0;foo"));
-m.put(BulkFileColumnFamily.NAME, new Text("/someFile"), new Value("5"));
-violations = mc.check(createEnv(), m);
-assertNotNull(violations);
-assertEquals(1, violations.size());
-assertEquals(Short.valueOf((short) 8), violations.get(0));
+m.put(
+BulkFileColumnFamily.NAME, StoredTabletFile
+.of(new
Path("hdfs://1.2.3.4/accumulo/tables/2a/t-0003/someFile")).getMetadataText(),
+new Value("5"));
+assertViolation(mc, m, (short) 8);
- // two active txids w/ files
+ // two files w/ same txid
m = new Mutation(new Text("0;foo"));
-m.put(BulkFileColumnFamily.NAME, new Text("/someFile"), new Value("5"));
-m.put(DataFileColumnFamily.NAME, new
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit b498c9797626b8d8eb9cdb9bbba246973463aafb
Merge: c650408d05 f8cb312032
Author: Christopher L. Shannon (cshannon)
AuthorDate: Sat Dec 2 09:52:34 2023 -0500
Merge branch '2.1'
.../accumulo/manager/TabletGroupWatcher.java | 71 ++
1 file changed, 45 insertions(+), 26 deletions(-)
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index c4e29eaf69,8171aa..068bb4930b
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@@ -615,131 -669,10 +613,130 @@@ abstract class TabletGroupWatcher exten
}
}
+ // Remove the merged marker from the last tablet in the merge range
+ private void clearMerged(MergeInfo mergeInfo, BatchWriter bw, HighTablet
highTablet)
+ throws AccumuloException {
+Manager.log.debug("Clearing MERGED marker for {}", mergeInfo.getExtent());
+var m = new Mutation(highTablet.getExtent().toMetaRow());
+MergedColumnFamily.MERGED_COLUMN.putDelete(m);
+bw.addMutation(m);
+bw.flush();
+ }
+
+ // This method finds returns the deletion starting row (exclusive) for
tablets that
+ // need to be actually deleted. If the startTablet is null then
+ // the deletion start row will just be null as all tablets are being deleted
+ // up to the end. Otherwise, this returns the endRow of the first tablet
+ // as the first tablet should be kept and will have been previously
+ // fenced if necessary
+ private Text getDeletionStartRow(final KeyExtent startTablet) {
+if (startTablet == null) {
+ Manager.log.debug("First tablet for delete range is null");
+ return null;
+}
+
+final Text deletionStartRow = startTablet.endRow();
+Manager.log.debug("Start row is {} for deletion", deletionStartRow);
+
+return deletionStartRow;
+ }
+
+ // This method finds returns the deletion ending row (inclusive) for
tablets that
+ // need to be actually deleted. If the endTablet is null then
+ // the deletion end row will just be null as all tablets are being deleted
+ // after the start row. Otherwise, this returns the prevEndRow of the last
tablet
+ // as the last tablet should be kept and will have been previously
+ // fenced if necessary
+ private Text getDeletionEndRow(final KeyExtent endTablet) {
+if (endTablet == null) {
+ Manager.log.debug("Last tablet for delete range is null");
+ return null;
+}
+
+Text deletionEndRow = endTablet.prevEndRow();
+Manager.log.debug("Deletion end row is {}", deletionEndRow);
+
+return deletionEndRow;
+ }
+
+ private static boolean isFirstTabletInTable(KeyExtent tablet) {
+return tablet != null && tablet.prevEndRow() == null;
+ }
+
+ private static boolean isLastTabletInTable(KeyExtent tablet) {
+return tablet != null && tablet.endRow() == null;
+ }
+
+ private static boolean areContiguousTablets(KeyExtent firstTablet,
KeyExtent lastTablet) {
+return firstTablet != null && lastTablet != null
+&& Objects.equals(firstTablet.endRow(), lastTablet.prevEndRow());
+ }
+
+ private boolean hasTabletsToDelete(final KeyExtent firstTabletInRange,
+ final KeyExtent lastTableInRange) {
+// If the tablets are equal (and not null) then the deletion range is
just part of 1 tablet
+// which will be fenced so there are no tablets to delete. The null check
is because if both
+// are null then we are just deleting everything, so we do have tablets
to delete
+if (Objects.equals(firstTabletInRange, lastTableInRange) &&
firstTabletInRange != null) {
+ Manager.log.trace(
+ "No tablets to delete, firstTablet {} equals lastTablet {} in
deletion range and was fenced.",
+ firstTabletInRange, lastTableInRange);
+ return false;
+ // If the lastTablet of the deletion range is the first tablet of the
table it has been fenced
+ // already so nothing to actually delete before it
+} else if (isFirstTabletInTable(lastTableInRange)) {
+ Manager.log.trace(
+ "No tablets to delete, lastTablet {} in deletion range is the first
tablet of the table and was fenced.",
+ lastTableInRange);
+ return false;
+ // If the firstTablet of the deletion range is the last tablet of the
table it has been fenced
+ // already so nothing to actually delete after it
+} else if (isLastTabletInTable(firstTabletInRange)) {
+ Manager.log.trace(
+ "No tablets to delete, firstTablet {} in deletion range is the last
tablet of the table and was fenced.",
+ firstTabletInRange);
+ return false;
+ // If the firstTablet and
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 6abb607d12216fd38057df66154df02bdb854f0c
Merge: a03d364632 751a110efe
Author: Keith Turner
AuthorDate: Thu Nov 30 16:19:24 2023 -0500
Merge branch '2.1'
.../org/apache/accumulo/tserver/tablet/Tablet.java | 10 +-
.../apache/accumulo/test/functional/BulkNewIT.java | 147 +
2 files changed, 152 insertions(+), 5 deletions(-)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index fc373ae30c,3f7eebe95f..1b6ba2f3cf
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@@ -1770,9 -1787,14 +1770,14 @@@ public class Tablet extends TabletBase
try {
tabletServer.updateBulkImportState(files, BulkImportState.LOADING);
- var storedTabletFile = getDatafileManager().importMapFiles(tid,
entries, setTime);
- lastMapFileImportTime = System.currentTimeMillis();
+ getDatafileManager().importDataFiles(tid, entries, setTime);
+ lastDataFileImportTime = System.currentTimeMillis();
+ synchronized (this) {
+ // only mark the bulk import a success if no exception was thrown
+ bulkImported.computeIfAbsent(tid, k -> new
ArrayList<>()).addAll(fileMap.keySet());
+ }
+
if (isSplitPossible()) {
getTabletServer().executeSplit(this);
} else {
diff --cc test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
index 5280d279dc,a8aebd2e25..6a0448749c
--- a/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
@@@ -61,7 -71,9 +71,10 @@@ import org.apache.accumulo.core.data.co
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVWriter;
import org.apache.accumulo.core.file.rfile.RFile;
+ import org.apache.accumulo.core.metadata.MetadataTable;
+ import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.accumulo.core.metadata.UnreferencedTabletFile;
+ import org.apache.accumulo.core.metadata.schema.MetadataSchema;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.security.Authorizations;
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
dlmarion pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit a03d36463257f1a911812f76015b22da2f33110a
Merge: a47ee0b77d e34f7fe1ce
Author: Dave Marion
AuthorDate: Thu Nov 30 18:11:37 2023 +
Merge branch '2.1'
.../org/apache/accumulo/core/metrics/MetricsUtil.java| 16 +---
.../accumulo/coordinator/CompactionCoordinator.java | 2 +-
.../java/org/apache/accumulo/compactor/Compactor.java| 2 +-
.../org/apache/accumulo/gc/SimpleGarbageCollector.java | 3 ++-
.../main/java/org/apache/accumulo/manager/Manager.java | 2 +-
.../java/org/apache/accumulo/tserver/ScanServer.java | 2 +-
.../java/org/apache/accumulo/tserver/TabletServer.java | 4 ++--
.../java/org/apache/accumulo/test/metrics/MetricsIT.java | 2 ++
8 files changed, 19 insertions(+), 14 deletions(-)
diff --cc
server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index ef84295540,88df289a56..cb7c4731bb
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@@ -266,8 -268,7 +266,8 @@@ public class CompactionCoordinator exte
try {
MetricsUtil.initializeMetrics(getContext().getConfiguration(),
this.applicationName,
- clientAddress);
+ clientAddress, getContext().getInstanceName());
+ MetricsUtil.initializeProducers(this);
} catch (ClassNotFoundException | InstantiationException |
IllegalAccessException
| IllegalArgumentException | InvocationTargetException |
NoSuchMethodException
| SecurityException e1) {
diff --cc
server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
index fc7d6a0234,9662757d3d..682437d298
---
a/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
+++
b/server/compactor/src/main/java/org/apache/accumulo/compactor/Compactor.java
@@@ -608,9 -616,7 +608,9 @@@ public class Compactor extends Abstract
try {
MetricsUtil.initializeMetrics(getContext().getConfiguration(),
this.applicationName,
- clientAddress);
+ clientAddress, getContext().getInstanceName());
+ pausedMetrics = new PausedCompactionMetrics();
+ MetricsUtil.initializeProducers(this, pausedMetrics);
} catch (ClassNotFoundException | InstantiationException |
IllegalAccessException
| IllegalArgumentException | InvocationTargetException |
NoSuchMethodException
| SecurityException e1) {
diff --cc
server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
index 01198752d7,0c22c4b50a..7152016d73
--- a/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
+++ b/server/gc/src/main/java/org/apache/accumulo/gc/SimpleGarbageCollector.java
@@@ -157,8 -168,9 +157,9 @@@ public class SimpleGarbageCollector ext
}
try {
- MetricsUtil.initializeMetrics(getContext().getConfiguration(),
this.applicationName, address);
+ MetricsUtil.initializeMetrics(getContext().getConfiguration(),
this.applicationName, address,
+ getContext().getInstanceName());
- MetricsUtil.initializeProducers(new GcMetrics(this));
+ MetricsUtil.initializeProducers(this, new GcMetrics(this));
} catch (ClassNotFoundException | InstantiationException |
IllegalAccessException
| IllegalArgumentException | InvocationTargetException |
NoSuchMethodException
| SecurityException e1) {
diff --cc server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index b75c8601c4,9d71e542f9..7d3806840f
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@@ -1103,9 -1099,8 +1103,9 @@@ public class Manager extends AbstractSe
try {
MetricsUtil.initializeMetrics(getContext().getConfiguration(),
this.applicationName,
- sa.getAddress());
+ sa.getAddress(), getContext().getInstanceName());
- ManagerMetrics.init(getConfiguration(), this);
+ ManagerMetrics mm = new ManagerMetrics(getConfiguration(), this);
+ MetricsUtil.initializeProducers(this, mm);
} catch (ClassNotFoundException | InstantiationException |
IllegalAccessException
| IllegalArgumentException | InvocationTargetException |
NoSuchMethodException
| SecurityException e1) {
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
index 8868065aa4,a34c2b1adc..fe227959f7
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
@@@ -370,9 -372,7 +370,9 @@@ public class ScanServer extends Abstrac
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 3e07ea58c6a56177c1dc1e67027115db29e87d45 Merge: 764f55e0c8 914f73630d Author: Keith Turner AuthorDate: Mon Nov 27 17:35:48 2023 -0500 Merge branch '2.1'
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit df362efeee3113d3e3c23ff6f6778f394a588363 Merge: 19bee3da4e 6921f79083 Author: Christopher Tubbs AuthorDate: Tue Nov 21 15:39:19 2023 -0500 Merge branch '2.1' pom.xml | 10 +++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --cc pom.xml index 7902225114,27eac387ab..0b772f5130 --- a/pom.xml +++ b/pom.xml @@@ -149,10 -150,10 +149,10 @@@ 1.1.1 1.76 5.5.0 - 2.20.0 + 2.23.0 3.3.6 +3.6.0 1.27.0 -2.0.9 2.0.7 0.17.0 3.8.3
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 19bee3da4ed15f147ad4de9ed46d93d633507075 Merge: 2d2a0190b9 6829e02d7f Author: Christopher Tubbs AuthorDate: Tue Nov 21 13:32:59 2023 -0500 Merge branch '2.1' shell/src/main/java/org/apache/accumulo/shell/commands/GrepCommand.java | 2 +- shell/src/main/java/org/apache/accumulo/shell/commands/ScanCommand.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 90821b115c90564004eb62625dab06864d4913be
Merge: b8089dd528 af53242538
Author: Keith Turner
AuthorDate: Mon Nov 20 21:13:32 2023 -0500
Merge branch '2.1'
.../accumulo/core/metrics/MetricsProducer.java | 150 ++---
.../accumulo/tserver/tablet/CompactableImpl.java | 11 +-
.../tablet/CompactableImplFileManagerTest.java | 34 +
3 files changed, 117 insertions(+), 78 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
index a2aa7d6a9a,1a8d618ce4..7aea9a7a6c
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
@@@ -47,14 -47,7 +47,14 @@@ import io.micrometer.core.instrument.Me
*
* N/A
* N/A
- * {@link #METRICS_LOW_MEMORY}
++ * {@value #METRICS_LOW_MEMORY}
+ * Guage
+ * reports 1 when process memory usage is above threshold, 0 when memory
is okay
+ *
+ *
+ * N/A
+ * N/A
- * {@link #METRICS_COMPACTOR_MAJC_STUCK}
+ * {@value #METRICS_COMPACTOR_MAJC_STUCK}
* LongTaskTimer
*
*
@@@ -431,20 -424,6 +431,20 @@@
* Counter
*
*
+ *
+ * N/A
+ * N/A
- * {@link #METRICS_SCAN_PAUSED_FOR_MEM}
++ * {@value #METRICS_SCAN_PAUSED_FOR_MEM}
+ * Counter
+ *
+ *
+ *
+ * N/A
+ * N/A
- * {@link #METRICS_SCAN_RETURN_FOR_MEM}
++ * {@value #METRICS_SCAN_RETURN_FOR_MEM}
+ * Counter
+ *
+ *
*
*
* {i|e}_{compactionServiceName}_{executor_name}_queued
@@@ -484,11 -456,40 +484,11 @@@
* Timer
*
*
- *
*
- * ReplicationQueue
- * Stat
- * {@value #METRICS_REPLICATION_QUEUE}
- * Timer
- *
- *
- *
- * ReplicationQueue10m
- * Quantiles
- * N/A
- * N/A
- *
- *
- *
- * filesPendingReplication
- * Stat
- * {@value #METRICS_REPLICATION_PENDING_FILES}
- * Gauge
*
- *
- *
- * maxReplicationThreads
- * Stat
- * {@value #METRICS_REPLICATION_THREADS}
- * Gauge
*
- * {@link #METRICS_MINC_PAUSED}
- *
- *
- * numPeers
- * Stat
- * {@value #METRICS_REPLICATION_PEERS}
- * Gauge
++ * {@value #METRICS_MINC_PAUSED}
+ * Counter
*
*
*
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index 8fe91dab5b,11722c100f..ce91f5d3ac
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@@ -872,11 -1081,10 +878,10 @@@ public class CompactableImpl implement
}
}
}
-
}
- static Collection asFileNames(Set files) {
-return Collections2.transform(files, StoredTabletFile::getFileName);
+ static Collection asMinimalString(Set files) {
+return Collections2.transform(files, StoredTabletFile::toMinimalString);
}
@Override
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 0e8dc7ab8495b0e34889f4a450fdf027a5f79bc4
Merge: 8f9636b268 db98f7d988
Author: Christopher Tubbs
AuthorDate: Tue Nov 14 11:34:48 2023 -0500
Merge branch '2.1'
.../org/apache/accumulo/shell/commands/GrepCommand.java | 16 ++--
.../org/apache/accumulo/shell/commands/ScanCommand.java | 10 +-
2 files changed, 19 insertions(+), 7 deletions(-)
diff --cc
shell/src/main/java/org/apache/accumulo/shell/commands/GrepCommand.java
index a5975d6e54,90c4a6fb44..2579fb4b01
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/GrepCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/GrepCommand.java
@@@ -49,7 -49,24 +49,21 @@@ public class GrepCommand extends ScanCo
if (cl.getArgList().isEmpty()) {
throw new MissingArgumentException("No terms specified");
}
+ // Configure formatting options
+ final FormatterConfig config = new FormatterConfig();
+ config.setPrintTimestamps(cl.hasOption(timestampOpt.getOpt()));
+ if (cl.hasOption(showFewOpt.getOpt())) {
+ final String showLength = cl.getOptionValue(showFewOpt.getOpt());
+ try {
+ final int length = Integer.parseInt(showLength);
+ config.setShownLength(length);
+ } catch (NumberFormatException nfe) {
+ Shell.log.error("Arg must be an integer.", nfe);
+ } catch (IllegalArgumentException iae) {
+ Shell.log.error("Arg must be greater than one.", iae);
+ }
+ }
final Class formatter = getFormatter(cl,
tableName, shellState);
- @SuppressWarnings("deprecation")
- final org.apache.accumulo.core.util.interpret.ScanInterpreter
interpeter =
- getInterpreter(cl, tableName, shellState);
// handle first argument, if present, the authorizations list to
// scan with
@@@ -81,11 -98,9 +95,9 @@@
}
try {
// handle columns
-fetchColumns(cl, scanner, interpeter);
+fetchColumns(cl, scanner);
// output the records
- final FormatterConfig config = new FormatterConfig();
- config.setPrintTimestamps(cl.hasOption(timestampOpt.getOpt()));
printRecords(cl, shellState, config, scanner, formatter, printFile);
} finally {
scanner.close();
diff --cc
shell/src/main/java/org/apache/accumulo/shell/commands/ScanCommand.java
index 2e8a7514d0,12ccc94350..a20b6f5c6d
--- a/shell/src/main/java/org/apache/accumulo/shell/commands/ScanCommand.java
+++ b/shell/src/main/java/org/apache/accumulo/shell/commands/ScanCommand.java
@@@ -56,9 -57,10 +56,10 @@@ import org.apache.hadoop.io.Text
public class ScanCommand extends Command {
- private Option scanOptAuths, scanOptRow, scanOptColumns,
disablePaginationOpt, showFewOpt,
- outputFileOpt, scanOptCf, scanOptCq;
- private Option scanOptAuths, scanOptRow, scanOptColumns,
disablePaginationOpt, formatterOpt,
- interpreterOpt, formatterInterpeterOpt, outputFileOpt, scanOptCf,
scanOptCq;
++ private Option scanOptAuths, scanOptRow, scanOptColumns,
disablePaginationOpt, outputFileOpt,
++ scanOptCf, scanOptCq;
+ protected Option showFewOpt;
protected Option timestampOpt;
protected Option profileOpt;
private Option optStartRowExclusive;
@@@ -388,7 -481,9 +388,6 @@@
o.addOption(timestampOpt);
o.addOption(disablePaginationOpt);
o.addOption(OptUtil.tableOpt("table to be scanned"));
- o.addOption(showFewOpt);
-o.addOption(formatterOpt);
-o.addOption(interpreterOpt);
-o.addOption(formatterInterpeterOpt);
o.addOption(timeoutOption);
if (Arrays.asList(ScanCommand.class.getName(),
GrepCommand.class.getName(),
EGrepCommand.class.getName()).contains(this.getClass().getName())) {
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit a4b4f540c839631a54746f977fc565272b29c020
Merge: 227f84fb48 4b63ead6cb
Author: Keith Turner
AuthorDate: Thu Nov 9 11:31:45 2023 -0500
Merge branch '2.1'
.../tserver/compactions/CompactionManager.java | 28 ++-
.../tserver/compactions/CompactionService.java | 21 +-
.../compactions/ProvisionalCompactionPlanner.java | 67 ++
.../compaction/BadCompactionServiceConfigIT.java | 233 +
4 files changed, 334 insertions(+), 15 deletions(-)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionManager.java
index 9957eb1dee,be872a6ed9..bbabc9f262
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionManager.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionManager.java
@@@ -53,7 -54,9 +54,9 @@@ import org.apache.accumulo.tserver.tabl
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
++import com.github.benmanes.caffeine.cache.Cache;
++import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.base.Preconditions;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
import com.google.common.collect.Sets;
public class CompactionManager {
@@@ -187,6 -209,9 +199,8 @@@
Map tmpServices = new HashMap<>();
-unknownCompactionServiceErrorCache =
-CacheBuilder.newBuilder().expireAfterWrite(5, MINUTES).build();
++unknownCompactionServiceErrorCache =
Caffeine.newBuilder().expireAfterWrite(5, MINUTES).build();
+
currentCfg.getPlanners().forEach((serviceName, plannerClassName) -> {
try {
tmpServices.put(CompactionServiceId.of(serviceName),
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit a4cc5f82c37b4ff1cb2bb44d99e0b7cef1302fc1 Merge: 3c4292d747 e34b439e7d Author: Daniel Roberts AuthorDate: Thu Nov 2 04:26:36 2023 + Merge branch '2.1' .../spi/compaction/DefaultCompactionPlanner.java | 5 ++-- .../test/functional/GarbageCollectorIT.java| 34 +- 2 files changed, 24 insertions(+), 15 deletions(-) diff --cc test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java index 73fb6d5665,b1372ac3ad..dc9fb34174 --- a/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/GarbageCollectorIT.java @@@ -32,10 -31,10 +32,9 @@@ import java.time.Duration import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.Map.Entry; - import java.util.Objects; -import java.util.concurrent.TimeUnit; +import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream;
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 3c4292d7476904f65da316c75889e417afb00c4e Merge: 4641c55e93 f79bf07950 Author: Christopher Tubbs AuthorDate: Mon Oct 30 13:06:25 2023 -0400 Merge branch '2.1' pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --cc pom.xml index 0370acc1f7,12e7379046..7902225114 --- a/pom.xml +++ b/pom.xml @@@ -151,11 -152,11 +151,11 @@@ 5.5.0 2.20.0 3.3.6 +3.6.0 1.27.0 -2.0.9 2.0.7 0.17.0 - 3.8.2 + 3.8.3
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 4641c55e933ba0efa647562ea2c63caab63defcd
Merge: 60f46e1abd a2ce072e5c
Author: Christopher L. Shannon (cshannon)
AuthorDate: Fri Oct 27 18:44:02 2023 -0400
Merge branch '2.1'
.../java/org/apache/accumulo/manager/Manager.java | 6 ++
.../accumulo/manager/TabletGroupWatcher.java | 27 +++-
.../apache/accumulo/manager/state/MergeStats.java | 23 +++
.../accumulo/manager/state/MergeStatsTest.java | 71 ++
.../apache/accumulo/test/manager/MergeStateIT.java | 19 ++
5 files changed, 145 insertions(+), 1 deletion(-)
diff --cc server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
index 009a407dd0,196b50d76c..8142af7c3f
--- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
+++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java
@@@ -648,8 -645,25 +648,14 @@@ public class Manager extends AbstractSe
case COMPLETE:
break;
case STARTED:
-case SPLITTING:
- return TabletGoalState.HOSTED;
-case WAITING_FOR_CHOPPED:
- if
(tls.getState(tserverSet.getCurrentServers()).equals(TabletState.HOSTED)) {
-if (tls.chopped) {
- return TabletGoalState.UNASSIGNED;
-}
- } else if (tls.chopped && tls.walogs.isEmpty()) {
-return TabletGoalState.UNASSIGNED;
- }
-
return TabletGoalState.HOSTED;
case WAITING_FOR_OFFLINE:
+ // If we have walogs we need to be HOSTED to recover
+ if (!tls.walogs.isEmpty()) {
+ return TabletGoalState.HOSTED;
+ } else {
+ return TabletGoalState.UNASSIGNED;
+ }
case MERGING:
return TabletGoalState.UNASSIGNED;
}
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
index bbee9aea48,4ffeee967d..b99bf3473b
---
a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java
@@@ -836,65 -801,15 +847,74 @@@ abstract class TabletGroupWatcher exten
Key key = entry.getKey();
Value value = entry.getValue();
+ // Verify that Tablet is offline
+ if (isTabletAssigned(key)) {
+ throw new IllegalStateException(
+ "Tablet " + key.getRow() + " is assigned during a merge!");
+ // Verify that Tablet has no WALs
+ } else if (key.getColumnFamily().equals(LogColumnFamily.NAME)) {
+ throw new IllegalStateException("Tablet " + key.getRow() + " has
walogs during a merge!");
-} else if (key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
- m.put(key.getColumnFamily(), key.getColumnQualifier(), value);
++}
++
+final KeyExtent keyExtent = KeyExtent.fromMetaRow(key.getRow());
+
+// Keep track of the last Key Extent seen so we can use it to fence
+// of RFiles when merging the metadata
+if (lastExtent != null && !keyExtent.equals(lastExtent)) {
+ previousKeyExtent = lastExtent;
+}
+
+// Special case to handle the highest/stop tablet, which is where
files are
+// merged to. The existing merge code won't delete files from this
tablet
+// so we need to handle the deletes in this tablet when fencing files.
+// We may be able to make this simpler in the future.
+if (keyExtent.equals(stopExtent)) {
+ if (previousKeyExtent != null
+ && key.getColumnFamily().equals(DataFileColumnFamily.NAME)) {
+
+// Fence off existing files by the end row of the previous tablet
and current tablet
+final StoredTabletFile existing =
StoredTabletFile.of(key.getColumnQualifier());
+// The end row should be inclusive for the current tablet and the
previous end row
+// should be exclusive for the start row
+Range fenced = new Range(previousKeyExtent.endRow(), false,
keyExtent.endRow(), true);
+
+// Clip range if exists
+fenced = existing.hasRange() ? existing.getRange().clip(fenced) :
fenced;
+
+final StoredTabletFile newFile =
StoredTabletFile.of(existing.getPath(), fenced);
+// If the existing metadata does not match then we need to delete
the old
+// and replace with a new range
+if (!existing.equals(newFile)) {
+ m.putDelete(DataFileColumnFamily.NAME,
existing.getMetadataText());
+ m.put(key.getColumnFamily(), newFile.getMetadataText(), value);
+}
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit e951e62d32dbcd66c31a9a4f881091c6a8d7f7b4 Merge: b5b48c3e38 75e959af26 Author: Christopher Tubbs AuthorDate: Thu Oct 26 22:31:00 2023 -0400 Merge branch '2.1' pom.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit c4c8c572c8be14d882899f8f9370700227e87423
Merge: c14a97fe8c e6cbdb4418
Author: Daniel Roberts
AuthorDate: Tue Oct 24 17:21:21 2023 +
Merge branch '2.1'
.../org/apache/accumulo/core/conf/Property.java| 243 ++---
.../org/apache/accumulo/core/gc/Reference.java | 7 +
.../accumulo/core/gc/ReferenceDirectory.java | 2 +-
.../org/apache/accumulo/core/gc/ReferenceFile.java | 32 ++-
.../apache/accumulo/core/conf/PropertyTest.java| 4 +
.../accumulo/server/gc/AllVolumesDirectory.java| 2 +-
.../accumulo/server/metadata/ServerAmpleImpl.java | 4 +-
.../accumulo/server/util/ManagerMetadataUtil.java | 4 +
.../accumulo/server/util/MetadataTableUtil.java| 2 +-
.../main/java/org/apache/accumulo/gc/GCRun.java| 31 +--
.../accumulo/gc/GarbageCollectionAlgorithm.java| 16 +-
.../apache/accumulo/gc/GarbageCollectionTest.java | 164 --
.../accumulo/manager/TabletGroupWatcher.java | 2 +-
.../tableOps/bulkVer2/CleanUpBulkImport.java | 2 +-
.../test/functional/GarbageCollectorIT.java| 2 +-
15 files changed, 296 insertions(+), 221 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 903c5d7443,bafc826c5a..58be89df05
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -60,28 -61,28 +60,28 @@@ public enum Property
@Sensitive
RPC_SSL_KEYSTORE_PASSWORD("rpc.javax.net.ssl.keyStorePassword", "",
PropertyType.STRING,
"Password used to encrypt the SSL private keystore. "
- + "Leave blank to use the Accumulo instance secret",
+ + "Leave blank to use the Accumulo instance secret.",
"1.6.0"),
RPC_SSL_KEYSTORE_TYPE("rpc.javax.net.ssl.keyStoreType", "jks",
PropertyType.STRING,
- "Type of SSL keystore", "1.6.0"),
+ "Type of SSL keystore.", "1.6.0"),
RPC_SSL_TRUSTSTORE_PATH("rpc.javax.net.ssl.trustStore", "",
PropertyType.PATH,
- "Path of the truststore file for the root cert", "1.6.0"),
+ "Path of the truststore file for the root cert.", "1.6.0"),
@Sensitive
RPC_SSL_TRUSTSTORE_PASSWORD("rpc.javax.net.ssl.trustStorePassword", "",
PropertyType.STRING,
- "Password used to encrypt the SSL truststore. Leave blank to use no
password", "1.6.0"),
+ "Password used to encrypt the SSL truststore. Leave blank to use no
password.", "1.6.0"),
RPC_SSL_TRUSTSTORE_TYPE("rpc.javax.net.ssl.trustStoreType", "jks",
PropertyType.STRING,
- "Type of SSL truststore", "1.6.0"),
+ "Type of SSL truststore.", "1.6.0"),
RPC_USE_JSSE("rpc.useJsse", "false", PropertyType.BOOLEAN,
"Use JSSE system properties to configure SSL rather than the " +
RPC_PREFIX.getKey()
- + "javax.net.ssl.* Accumulo properties",
+ + "javax.net.ssl.* Accumulo properties.",
"1.6.0"),
RPC_SSL_CIPHER_SUITES("rpc.ssl.cipher.suites", "", PropertyType.STRING,
- "Comma separated list of cipher suites that can be used by accepted
connections", "1.6.1"),
+ "Comma separated list of cipher suites that can be used by accepted
connections.", "1.6.1"),
- RPC_SSL_ENABLED_PROTOCOLS("rpc.ssl.server.enabled.protocols", "TLSv1.2",
PropertyType.STRING,
+ RPC_SSL_ENABLED_PROTOCOLS("rpc.ssl.server.enabled.protocols", "TLSv1.3",
PropertyType.STRING,
- "Comma separated list of protocols that can be used to accept
connections", "1.6.2"),
+ "Comma separated list of protocols that can be used to accept
connections.", "1.6.2"),
- RPC_SSL_CLIENT_PROTOCOL("rpc.ssl.client.protocol", "TLSv1.2",
PropertyType.STRING,
+ RPC_SSL_CLIENT_PROTOCOL("rpc.ssl.client.protocol", "TLSv1.3",
PropertyType.STRING,
- "The protocol used to connect to a secure server, must be in the list
of enabled protocols "
- + "on the server side (rpc.ssl.server.enabled.protocols)",
+ "The protocol used to connect to a secure server. Must be in the list
of enabled protocols "
+ + "on the server side `rpc.ssl.server.enabled.protocols`.",
"1.6.2"),
RPC_SASL_QOP("rpc.sasl.qop", "auth", PropertyType.STRING,
"The quality of protection to be used with SASL. Valid values are
'auth', 'auth-int',"
@@@ -250,7 -265,12 +250,7 @@@
"Enables tracing functionality using OpenTelemetry (assuming
OpenTelemetry is configured).",
"2.1.0"),
GENERAL_THREADPOOL_SIZE("general.server.threadpool.size", "1",
PropertyType.COUNT,
- "The number of threads to use for server-internal scheduled tasks",
"2.1.0"),
+ "The number of threads to use for server-internal scheduled tasks.",
"2.1.0"),
- @Deprecated(since = "2.1.0")
- @ReplacedBy(property = GENERAL_THREADPOOL_SIZE)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit bc105dc66f1326597e9cc225f1294b34ef7a01a5 Merge: 62c6eb1c75 9dfa9e3b64 Author: Christopher Tubbs AuthorDate: Sat Oct 21 13:28:41 2023 -0400 Merge branch '2.1'
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 85b7d3be9f7c8942fe2ecd2c4a97e335cb150c8b Merge: 4de6087f27 324f52b711 Author: Daniel Roberts AuthorDate: Tue Oct 17 17:27:03 2023 + Merge branch '2.1' .../accumulo/tserver/TabletClientHandler.java | 3 + .../accumulo/tserver/tablet/DatafileManager.java | 53 +- .../accumulo/tserver/tablet/ScanDataSource.java| 108 + 3 files changed, 99 insertions(+), 65 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit f203043e831391496a94210c8e10ebd9c805cb40 Merge: d20b5a8cde 719697786c Author: Christopher Tubbs AuthorDate: Fri Oct 13 17:43:31 2023 -0400 Merge branch '2.1' LICENSE| 11 +- assemble/src/main/resources/LICENSE| 25 + .../core/clientImpl/TableOperationsImpl.java |2 +- .../org/apache/accumulo/core/data/Mutation.java|4 +- .../miniclusterImpl/MiniAccumuloClusterImpl.java |2 +- pom.xml|6 +- .../java/org/apache/accumulo/manager/Manager.java |5 +- .../manager/ManagerClientServiceHandler.java |2 +- .../manager/tableOps/compact/CompactionDriver.java |3 +- .../src/main/appended-resources/META-INF/LICENSE | 11 +- .../accumulo/monitor/resources/css/screen.css |3 +- .../external/bootstrap/css/bootstrap-icons.css | 2078 .../external/bootstrap/fonts/arrow-repeat.svg |4 - .../external/bootstrap/fonts/bootstrap-icons.woff | Bin 0 -> 176200 bytes .../external/bootstrap/fonts/bootstrap-icons.woff2 | Bin 0 -> 130608 bytes .../bootstrap/fonts/three-dots-vertical.svg|3 - .../apache/accumulo/monitor/resources/js/navbar.js |1 + .../accumulo/monitor/resources/js/systemAlert.js | 172 ++ .../apache/accumulo/monitor/templates/default.ftl |3 + .../org/apache/accumulo/monitor/templates/ec.ftl |4 +- .../apache/accumulo/monitor/templates/navbar.ftl |5 +- .../accumulo/monitor/templates/systemAlert.ftl | 31 + .../accumulo/tserver/TabletClientHandler.java |3 +- .../org/apache/accumulo/tserver/log/DfsLogger.java |2 +- .../org/apache/accumulo/tserver/log/LogSorter.java |4 +- .../accumulo/tserver/logger/LogFileValue.java |2 +- .../apache/accumulo/tserver/logger/LogReader.java |2 +- .../accumulo/shell/commands/GetAuthsCommand.java |4 +- test/pom.xml |4 +- .../test/compaction/CompactionExecutorIT.java |2 +- .../accumulo/test/functional/CompactionIT.java |2 +- 31 files changed, 2365 insertions(+), 35 deletions(-) diff --cc server/manager/src/main/java/org/apache/accumulo/manager/Manager.java index d59ca8d552,b86cba576c..009a407dd0 --- a/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/Manager.java @@@ -18,7 -18,7 +18,8 @@@ */ package org.apache.accumulo.manager; +import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly; + import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Collections.emptySortedMap; import static java.util.concurrent.TimeUnit.HOURS; import static java.util.concurrent.TimeUnit.MILLISECONDS; diff --cc test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java index 928810fbe1,9bff345b17..6e48bf1cf0 --- a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java @@@ -30,8 -28,8 +30,9 @@@ import static org.junit.jupiter.api.Ass import java.io.IOException; import java.time.Duration; import java.util.ArrayList; + import java.util.Arrays; import java.util.EnumSet; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry;
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 19b4c508bf316671af4158d46d49b29eed9caf48 Merge: b9fdbc3beb 84d034589d Author: Christopher Tubbs AuthorDate: Thu Sep 28 20:03:47 2023 -0400 Merge branch '2.1' .../rpc/ProtocolOverridingSSLSocketFactory.java| 111 - .../accumulo/core/rpc/SslConnectionParams.java | 20 +--- .../org/apache/accumulo/core/rpc/ThriftUtil.java | 108 +--- .../apache/accumulo/server/rpc/TServerUtils.java | 2 +- 4 files changed, 8 insertions(+), 233 deletions(-) diff --cc core/src/main/java/org/apache/accumulo/core/rpc/ThriftUtil.java index 5be5ce82aa,6924b4c862..dd44653dd3 --- a/core/src/main/java/org/apache/accumulo/core/rpc/ThriftUtil.java +++ b/core/src/main/java/org/apache/accumulo/core/rpc/ThriftUtil.java @@@ -18,14 -18,11 +18,12 @@@ */ package org.apache.accumulo.core.rpc; +import static org.apache.accumulo.core.util.LazySingletons.RANDOM; + - import java.io.FileInputStream; import java.io.IOException; import java.io.UncheckedIOException; import java.net.InetAddress; import java.nio.channels.ClosedByInterruptException; - import java.security.KeyStore; -import java.security.SecureRandom; import java.util.HashMap; import java.util.Map; @@@ -52,10 -44,6 +44,8 @@@ import org.apache.thrift.transport.TTra import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.net.HostAndPort; + - import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - /** * Factory methods for creating Thrift client objects */
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 5488a7c74b8b1dc29df37be5dcaa7bace658ba10
Merge: 9aa08ef6fa 27b76c70b8
Author: Christopher Tubbs
AuthorDate: Thu Sep 28 17:10:03 2023 -0400
Merge branch '2.1'
This merge retains the changes for automatic module names from commit
f5bbd894bc2248f7d6eb2e99f94c154b8f9097eb for PR #2498 that were reverted
by commit 27b76c70b8290f0354c2a8719f0658aa89281a1d in the 2.1 branch, as
explained in that commit.
pom.xml | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --cc pom.xml
index 38a5a5f920,68ed4d64ad..a92287c8f9
--- a/pom.xml
+++ b/pom.xml
@@@ -151,6 -152,6 +151,7 @@@
5.5.0
2.20.0
3.3.6
++3.6.0
1.27.0
2.0.9
2.0.7
@@@ -803,8 -821,8 +804,7 @@@
--
--
++
${accumulo.module.name}
${mvngit.commit.id}
true
@@@ -821,6 -839,6 +821,7 @@@
true
-J-Xmx512m
all,-missing
++true
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 5aa53a8e0cd75c4f612785dc4cf8e5946e790ea0 Merge: 1ab0cc7808 af47a2f0bd Author: Christopher Tubbs AuthorDate: Mon Sep 25 14:52:15 2023 -0400 Merge branch '2.1' pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit ff916e6759fbfd88d09b2ff0b3681d082c1e342e
Merge: 7005006620 2262125874
Author: Christopher Tubbs
AuthorDate: Wed Sep 20 14:13:08 2023 -0400
Merge branch '2.1'
LICENSE| 13 +-
.../src/main/appended-resources/META-INF/LICENSE | 13 +-
.../resources/external/bootstrap/css/bootstrap.css | 7217
.../external/bootstrap/css/bootstrap.css.map |1 -
.../external/bootstrap/js/bootstrap.bundle.js | 6314 +
.../resources/external/bootstrap/js/bootstrap.js | 5016 --
.../external/bootstrap/js/bootstrap.js.map |1 -
.../datatables/css/dataTables.bootstrap5.css | 87 +-
.../external/datatables/css/jquery.dataTables.css | 578 --
.../external/datatables/images/sort_asc.png| Bin 160 -> 0 bytes
.../datatables/images/sort_asc_disabled.png| Bin 148 -> 0 bytes
.../external/datatables/images/sort_both.png | Bin 201 -> 0 bytes
.../external/datatables/images/sort_desc.png | Bin 158 -> 0 bytes
.../datatables/images/sort_desc_disabled.png | Bin 146 -> 0 bytes
.../datatables/js/dataTables.bootstrap5.js | 86 +-
.../external/datatables/js/jquery.dataTables.js| 235 +-
.../external/flot/jquery.flot.drawSeries.js| 123 +-
.../jquery/{jquery-3.6.1.js => jquery-3.7.1.js}| 1915 +++---
.../apache/accumulo/monitor/templates/default.ftl |4 +-
19 files changed, 11791 insertions(+), 9812 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 48d69f30a18da524e03ad068f733906e7428c5ea Merge: f7e59d3653 6c34c848c4 Author: Christopher Tubbs AuthorDate: Mon Sep 18 14:44:33 2023 -0400 Merge branch '2.1' .../java/org/apache/accumulo/tserver/tablet/DatafileManager.java | 5 ++--- .../src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit f7e59d3653edddc9bc4554fea5dbcba386d04fab Merge: e4127a5014 75140d8965 Author: Daniel Roberts AuthorDate: Mon Sep 18 18:24:46 2023 + Merge branch '2.1' .../main/java/org/apache/accumulo/gc/GCRun.java| 24 +- 1 file changed, 14 insertions(+), 10 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 688d2f58be12e40a2818ebc66376bd02a818e6b4
Merge: 59707d6002 de0cf9f739
Author: Keith Turner
AuthorDate: Thu Sep 14 18:40:38 2023 -0400
Merge branch '2.1'
.../accumulo/tserver/compactions/Compactable.java | 5 +-
.../compactions/InternalCompactionExecutor.java| 43 -
.../accumulo/tserver/tablet/CompactableImpl.java | 16 ++--
.../accumulo/tserver/tablet/DatafileManager.java | 15 ++-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 21 ++---
.../test/compaction/CompactionConfigChangeIT.java | 103 +
6 files changed, 176 insertions(+), 27 deletions(-)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index 3561904912,1a820bee5b..fb510c1eff
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@@ -1277,8 -1292,8 +1279,8 @@@ public class CompactableImpl implement
try {
TabletLogger.compacting(getExtent(), job, cInfo.localCompactionCfg);
tablet.incrementStatusMajor();
- var check = new CompactionCheck(service, kind, cInfo.checkCompactionId);
+ var check = new CompactionCheck(service, kind, keepRunning,
cInfo.checkCompactionId);
- TabletFile tmpFileName =
tablet.getNextMapFilenameForMajc(cInfo.propagateDeletes);
+ ReferencedTabletFile tmpFileName =
tablet.getNextDataFilenameForMajc(cInfo.propagateDeletes);
var compactEnv = new MajCEnv(kind, check, readLimiter, writeLimiter,
cInfo.propagateDeletes);
SortedMap allFiles =
tablet.getDatafiles();
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
index 97cb8d23d7,07be1a70bc..2cba023e8f
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
@@@ -348,8 -358,48 +357,8 @@@ class DatafileManager
// before the following metadata write is made
newFile =
tablet.updateTabletDataFile(commitSession.getMaxCommittedTime(), newDatafile,
dfv,
unusedWalLogs, flushId);
-
-// Mark that we have data we want to replicate
-// This WAL could still be in use by other Tablets *from the same
table*, so we can only
-// mark
-// that there is data to replicate,
-// but it is *not* closed. We know it is not closed by the fact that
this MinC triggered. A
-// MinC cannot happen unless the
-// tablet is online and thus these WALs are referenced by that
tablet. Therefore, the WAL
-// replication status cannot be 'closed'.
-@SuppressWarnings("deprecation")
-boolean replicate =
org.apache.accumulo.core.replication.ReplicationConfigurationUtil
-.isEnabled(tablet.getExtent(), tablet.getTableConfiguration());
-if (replicate) {
- // unusedWalLogs is of the form host/fileURI, need to strip off the
host portion
- Set logFileOnly = new HashSet<>();
- for (String unusedWalLog : unusedWalLogs) {
-int index = unusedWalLog.indexOf('/');
-if (index == -1) {
- log.warn(
- "Could not find host component to strip from DFSLogger
representation of WAL");
-} else {
- unusedWalLog = unusedWalLog.substring(index + 1);
-}
-logFileOnly.add(unusedWalLog);
- }
-
- if (log.isDebugEnabled()) {
-log.debug("Recording that data has been ingested into {} using
{}", tablet.getExtent(),
-logFileOnly);
- }
- for (String logFile : logFileOnly) {
-@SuppressWarnings("deprecation")
-Status status =
-
org.apache.accumulo.server.replication.StatusUtil.openWithUnknownLength();
-ReplicationTableUtil.updateFiles(tablet.getContext(),
tablet.getExtent(), logFile,
-status);
- }
-}
-
-tablet.finishClearingUnusedLogs();
} finally {
- tablet.finishClearingUnusedLogs();
+ tablet.getLogLock().unlock();
}
do {
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 59707d6002b37defbbf29c536ab1af054627102c
Merge: 3409d958bb 11824cf7a5
Author: Christopher Tubbs
AuthorDate: Thu Sep 14 17:13:03 2023 -0400
Merge branch '2.1'
.../accumulo/core/conf/AccumuloConfiguration.java | 4 +
.../org/apache/accumulo/core/conf/Property.java| 5 +
.../apache/accumulo/server/rpc/TServerUtils.java | 63 +++--
.../AccumuloConfigurationIsPropertySetTest.java| 302 +
.../apache/accumulo/gc/SimpleGarbageCollector.java | 3 +-
.../accumulo/test/functional/ZombieTServer.java| 10 +-
.../accumulo/test/performance/NullTserver.java | 3 +-
7 files changed, 360 insertions(+), 30 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
index f1131894b5,636b18db68..8a6e48ba86
---
a/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
+++
b/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
@@@ -424,8 -430,49 +424,12 @@@ public abstract class AccumuloConfigura
}
}
+ /**
+* @param prop Property to check
+* @return true if the given property has explicitly been set by a user,
false otherwise
+*/
public abstract boolean isPropertySet(Property prop);
- // deprecation property warning could get spammy in tserver so only warn
once
- boolean depPropWarned = false;
-
- @SuppressWarnings("deprecation")
- Integer getDeprecatedScanThreads(String name, boolean isScanServer) {
-
-Property prop;
-Property deprecatedProp;
-
-if (name.equals(SimpleScanDispatcher.DEFAULT_SCAN_EXECUTOR_NAME)) {
- prop = isScanServer ? Property.SSERV_SCAN_EXECUTORS_DEFAULT_THREADS
- : Property.TSERV_SCAN_EXECUTORS_DEFAULT_THREADS;
- deprecatedProp = Property.TSERV_READ_AHEAD_MAXCONCURRENT;
-} else if (name.equals("meta")) {
- prop = isScanServer ? Property.SSERV_SCAN_EXECUTORS_META_THREADS
- : Property.TSERV_SCAN_EXECUTORS_META_THREADS;
- deprecatedProp = Property.TSERV_METADATA_READ_AHEAD_MAXCONCURRENT;
-} else {
- return null;
-}
-
-if (!isPropertySet(prop) && isPropertySet(deprecatedProp)) {
- if (!depPropWarned) {
-depPropWarned = true;
-log.warn("Property {} is deprecated, use {} instead.",
deprecatedProp.getKey(),
-prop.getKey());
- }
- return Integer.valueOf(get(deprecatedProp));
-} else if (isPropertySet(prop) && isPropertySet(deprecatedProp) &&
!depPropWarned) {
- depPropWarned = true;
- log.warn("Deprecated property {} ignored because {} is set",
deprecatedProp.getKey(),
- prop.getKey());
-}
-
-return null;
- }
-
private static class RefCount {
T obj;
long count;
diff --cc
test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
index 2a8d023685,61c3c24e7c..2a8abcfffb
--- a/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/ZombieTServer.java
@@@ -26,21 -27,24 +26,22 @@@ import java.util.UUID
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.clientImpl.thrift.ClientService;
+import org.apache.accumulo.core.clientImpl.thrift.TInfo;
+ import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.conf.SiteConfiguration;
-import org.apache.accumulo.core.fate.zookeeper.ServiceLock;
-import org.apache.accumulo.core.fate.zookeeper.ServiceLock.LockLossReason;
-import org.apache.accumulo.core.fate.zookeeper.ServiceLock.LockWatcher;
import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.core.fate.zookeeper.ZooUtil.NodeExistsPolicy;
-import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.core.lock.ServiceLock;
+import org.apache.accumulo.core.lock.ServiceLock.LockLossReason;
+import org.apache.accumulo.core.lock.ServiceLock.LockWatcher;
+import org.apache.accumulo.core.lock.ServiceLockData;
+import org.apache.accumulo.core.lock.ServiceLockData.ThriftService;
+import org.apache.accumulo.core.manager.thrift.TabletServerStatus;
import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
import org.apache.accumulo.core.securityImpl.thrift.TCredentials;
-import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
-import org.apache.accumulo.core.tabletserver.thrift.TabletScanClientService;
+import org.apache.accumulo.core.tabletscan.thrift.TabletScanClientService;
+import org.apache.accumulo.core.tabletserver.thrift.TabletServerClientService;
import org.apache.accumulo.core.trace.TraceUtil;
-import org.apache.accumulo.core.trace.thrift.TInfo;
-import org.apache.accumulo.core.util.HostAndPort;
-import
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 3409d958bb1d69baf5b93bb8a588a3e3ee9aab1c Merge: 770c05b46a b1b2557f94 Author: Christopher Tubbs AuthorDate: Fri Sep 1 12:06:00 2023 -0400 Merge branch '2.1' .../TabletServerBatchReaderIterator.java | 11 +- .../core/clientImpl/TabletServerBatchWriter.java | 2 +- .../core/rpc/AccumuloTFramedTransportFactory.java | 58 + .../org/apache/accumulo/core/rpc/ThriftUtil.java | 7 +- .../apache/accumulo/core/rpc/ThriftUtilTest.java | 132 + .../test/functional/ThriftMaxFrameSizeIT.java | 115 -- .../java/org/apache/accumulo/test/util/Wait.java | 2 +- 7 files changed, 282 insertions(+), 45 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 770c05b46a4e3c135c85d0772ebb2b02d9db5414
Merge: 1b6dfa7915 c78dfc5235
Author: Christopher Tubbs
AuthorDate: Thu Aug 31 07:23:58 2023 -0400
Merge branch '2.1'
.../core/clientImpl/TabletServerBatchWriter.java | 260 +++--
.../accumulo/tserver/TabletClientHandler.java | 97
.../apache/accumulo/test/WriteAfterCloseIT.java| 203
3 files changed, 445 insertions(+), 115 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchWriter.java
index e0dd94d216,de66339886..cf6821c649
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchWriter.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletServerBatchWriter.java
@@@ -66,12 -71,15 +72,13 @@@ import org.apache.accumulo.core.dataImp
import org.apache.accumulo.core.dataImpl.TabletIdImpl;
import org.apache.accumulo.core.dataImpl.thrift.TMutation;
import org.apache.accumulo.core.dataImpl.thrift.UpdateErrors;
-import org.apache.accumulo.core.fate.zookeeper.ServiceLock;
++import org.apache.accumulo.core.lock.ServiceLock;
import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
- import
org.apache.accumulo.core.tabletingest.thrift.ConstraintViolationException;
+import org.apache.accumulo.core.tabletingest.thrift.TabletIngestClientService;
- import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
+ import org.apache.accumulo.core.tabletserver.thrift.NoSuchScanIDException;
-import org.apache.accumulo.core.tabletserver.thrift.TabletClientService;
import org.apache.accumulo.core.trace.TraceUtil;
-import org.apache.accumulo.core.trace.thrift.TInfo;
-import org.apache.accumulo.core.util.HostAndPort;
+ import org.apache.accumulo.core.util.Retry;
import org.apache.accumulo.core.util.threads.ThreadPools;
import org.apache.accumulo.core.util.threads.Threads;
import org.apache.thrift.TApplicationException;
@@@ -931,15 -945,17 +945,17 @@@ public class TabletServerBatchWriter im
timeoutTracker.startingWrite();
+ // If there is an open session, must close it before the batchwriter
closes or writes could
+ // happen after the batch writer closes. See #3721
try {
final HostAndPort parsedServer = HostAndPort.fromString(location);
-final TabletClientService.Iface client;
+final TabletIngestClientService.Iface client;
if (timeoutTracker.getTimeOut() < context.getClientTimeoutInMillis())
{
- client = ThriftUtil.getClient(ThriftClientTypes.TABLET_SERVER,
parsedServer, context,
+ client = ThriftUtil.getClient(ThriftClientTypes.TABLET_INGEST,
parsedServer, context,
timeoutTracker.getTimeOut());
} else {
- client = ThriftUtil.getClient(ThriftClientTypes.TABLET_SERVER,
parsedServer, context);
+ client = ThriftUtil.getClient(ThriftClientTypes.TABLET_INGEST,
parsedServer, context);
}
try {
@@@ -1038,6 -1048,114 +1048,114 @@@
throw new IOException(e);
}
}
+
+ class SessionCloser implements AutoCloseable {
+
+ private final String location;
+ private OptionalLong usid;
+
+ SessionCloser(String location) {
+ this.location = location;
+ usid = OptionalLong.empty();
+ }
+
+ void setSession(long usid) {
+ this.usid = OptionalLong.of(usid);
+ }
+
+ public long getSession() {
+ return usid.getAsLong();
+ }
+
+ void clearSession() {
+ usid = OptionalLong.empty();
+ }
+
+ @Override
+ public void close() throws ThriftSecurityException {
+ if (usid.isPresent()) {
+ try {
+ closeSession();
+ } catch (InterruptedException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ }
+
+ /**
+* Checks if there is a lock held by a tserver at a specific host and
port.
+*/
+ private boolean isALockHeld(String tserver) {
+ var root = context.getZooKeeperRoot() + Constants.ZTSERVERS;
+ var zLockPath = ServiceLock.path(root + "/" + tserver);
+ return ServiceLock.getSessionId(context.getZooCache(), zLockPath) !=
0;
+ }
+
+ private void closeSession() throws InterruptedException,
ThriftSecurityException {
+
+ Retry retry = Retry.builder().infiniteRetries().retryAfter(100,
MILLISECONDS)
+ .incrementBy(100, MILLISECONDS).maxWait(60,
SECONDS).backOffFactor(1.5)
+ .logInterval(3, MINUTES).createRetry();
+
+ final HostAndPort parsedServer = HostAndPort.fromString(location);
+
+ long startTime =
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 1b6dfa791554d08d6ae618dffe533925d4eed7c3
Merge: 81c6ae1657 f76f12baa6
Author: Christopher Tubbs
AuthorDate: Thu Aug 31 06:59:31 2023 -0400
Merge branch '2.1'
This completes the removal of the single-mutation Writer class
This fixes #3735
.../apache/accumulo/core/clientImpl/Writer.java| 122 -
.../accumulo/server/problems/ProblemReport.java| 10 +-
.../accumulo/server/problems/ProblemReports.java | 5 +-
.../accumulo/server/util/MetadataTableUtil.java| 51 +++--
.../org/apache/accumulo/test/BatchWriterIT.java| 95
.../accumulo/test/MetaConstraintRetryIT.java | 13 ++-
6 files changed, 125 insertions(+), 171 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/clientImpl/Writer.java
index 4a358aee6f,9f124ede54..00
deleted file mode 100644,100644
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/Writer.java
+++ /dev/null
@@@ -1,122 -1,55 +1,0 @@@
--/*
-- * Licensed to the Apache Software Foundation (ASF) under one
-- * or more contributor license agreements. See the NOTICE file
-- * distributed with this work for additional information
-- * regarding copyright ownership. The ASF licenses this file
-- * to you under the Apache License, Version 2.0 (the
-- * "License"); you may not use this file except in compliance
-- * with the License. You may obtain a copy of the License at
-- *
-- * https://www.apache.org/licenses/LICENSE-2.0
-- *
-- * Unless required by applicable law or agreed to in writing,
-- * software distributed under the License is distributed on an
-- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-- * KIND, either express or implied. See the License for the
-- * specific language governing permissions and limitations
-- * under the License.
-- */
--package org.apache.accumulo.core.clientImpl;
--
--import static com.google.common.base.Preconditions.checkArgument;
- import static
com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
- import static java.nio.charset.StandardCharsets.UTF_8;
- import static java.util.concurrent.TimeUnit.MILLISECONDS;
--
--import org.apache.accumulo.core.client.AccumuloException;
- import org.apache.accumulo.core.client.AccumuloSecurityException;
--import org.apache.accumulo.core.client.TableNotFoundException;
- import org.apache.accumulo.core.clientImpl.TabletLocator.TabletLocation;
- import org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException;
--import org.apache.accumulo.core.data.Mutation;
--import org.apache.accumulo.core.data.TableId;
- import org.apache.accumulo.core.dataImpl.KeyExtent;
- import org.apache.accumulo.core.rpc.ThriftUtil;
- import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
- import
org.apache.accumulo.core.tabletingest.thrift.ConstraintViolationException;
- import org.apache.accumulo.core.tabletingest.thrift.TDurability;
- import org.apache.accumulo.core.tabletingest.thrift.TabletIngestClientService;
- import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
- import org.apache.accumulo.core.trace.TraceUtil;
- import org.apache.hadoop.io.Text;
- import org.apache.thrift.TException;
- import org.apache.thrift.TServiceClient;
- import org.slf4j.Logger;
- import org.slf4j.LoggerFactory;
-
- import com.google.common.net.HostAndPort;
-import org.apache.accumulo.core.metadata.schema.Ample;
--
--public class Writer {
-
- private static final Logger log = LoggerFactory.getLogger(Writer.class);
--
-- private ClientContext context;
-- private TableId tableId;
--
-- public Writer(ClientContext context, TableId tableId) {
--checkArgument(context != null, "context is null");
--checkArgument(tableId != null, "tableId is null");
--this.context = context;
--this.tableId = tableId;
- }
-
- private static void updateServer(ClientContext context, Mutation m,
KeyExtent extent,
- HostAndPort server) throws TException, NotServingTabletException,
- ConstraintViolationException, AccumuloSecurityException {
- checkArgument(m != null, "m is null");
- checkArgument(extent != null, "extent is null");
- checkArgument(server != null, "server is null");
- checkArgument(context != null, "context is null");
-
- TabletIngestClientService.Iface client = null;
- try {
- client = ThriftUtil.getClient(ThriftClientTypes.TABLET_INGEST, server,
context);
- client.update(TraceUtil.traceInfo(), context.rpcCreds(),
extent.toThrift(), m.toThrift(),
- TDurability.DEFAULT);
- } catch (ThriftSecurityException e) {
- throw new AccumuloSecurityException(e.user, e.code);
- } finally {
- ThriftUtil.returnClient((TServiceClient) client, context);
- }
-- }
--
- public void update(Mutation m)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 81c6ae16570ae1af23a91947b28fa2c673279b80
Merge: 5bec1c33af f02d2ac500
Author: Christopher Tubbs
AuthorDate: Tue Aug 29 15:27:35 2023 -0400
Merge branch '2.1'
.../org/apache/accumulo/core/conf/Property.java| 17 -
.../server/tablets/UniqueNameAllocator.java| 43 ++
2 files changed, 28 insertions(+), 32 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 1261d83b35,0bff139725..7abc163f1f
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -1445,10 -1833,13 +1444,10 @@@ public enum Property
return key.startsWith(Property.TABLE_PREFIX.getKey())
|| key.startsWith(Property.TSERV_PREFIX.getKey())
|| key.startsWith(Property.MANAGER_PREFIX.getKey())
-|| key.startsWith(Property.MASTER_PREFIX.getKey())
|| key.startsWith(Property.GC_PREFIX.getKey())
|| key.startsWith(Property.GENERAL_ARBITRARY_PROP_PREFIX.getKey())
- || key.equals(Property.GENERAL_FILENAME_BASE_ALLOCATION.getKey())
- || key.equals(Property.GENERAL_FILENAME_JITTER_ALLOCATION.getKey());
+ ||
key.equals(Property.GENERAL_FILE_NAME_ALLOCATION_BATCH_SIZE_MIN.getKey())
-||
key.equals(Property.GENERAL_FILE_NAME_ALLOCATION_BATCH_SIZE_MAX.getKey())
-|| key.startsWith(VFS_CONTEXT_CLASSPATH_PROPERTY.getKey())
-|| key.startsWith(REPLICATION_PREFIX.getKey());
++||
key.equals(Property.GENERAL_FILE_NAME_ALLOCATION_BATCH_SIZE_MAX.getKey());
}
/**
diff --cc
server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
index 5e4cc77a7d,cd633137c5..14026124df
---
a/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
@@@ -19,9 -19,11 +19,10 @@@
package org.apache.accumulo.server.tablets;
import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.security.SecureRandom;
+import static org.apache.accumulo.core.util.LazySingletons.RANDOM;
import org.apache.accumulo.core.Constants;
+ import org.apache.accumulo.core.conf.DefaultConfiguration;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.util.FastFormat;
import org.apache.accumulo.server.ServerContext;
@@@ -36,15 -38,17 +37,16 @@@ import org.slf4j.LoggerFactory
*/
public class UniqueNameAllocator {
- private static Logger log =
LoggerFactory.getLogger(UniqueNameAllocator.class);
+ private static final Logger log =
LoggerFactory.getLogger(UniqueNameAllocator.class);
- private static final SecureRandom random = new SecureRandom();
+ private static final Property MIN_PROP =
Property.GENERAL_FILE_NAME_ALLOCATION_BATCH_SIZE_MIN;
+ private static final Property MAX_PROP =
Property.GENERAL_FILE_NAME_ALLOCATION_BATCH_SIZE_MAX;
+ private static final int DEFAULT_MIN =
DefaultConfiguration.getInstance().getCount(MIN_PROP);
- private static final int DEFAULT_BASE_ALLOCATION =
-
Integer.parseInt(Property.GENERAL_FILENAME_BASE_ALLOCATION.getDefaultValue());
+ private final ServerContext context;
+ private final String nextNamePath;
- private ServerContext context;
private long next = 0;
private long maxAllocated = 0;
- private String nextNamePath;
public UniqueNameAllocator(ServerContext context) {
this.context = context;
@@@ -75,25 -76,23 +74,23 @@@
}
private int getAllocation() {
- int baseAllocation =
-
context.getConfiguration().getCount(Property.GENERAL_FILENAME_BASE_ALLOCATION);
- int jitterAllocation =
-
context.getConfiguration().getCount(Property.GENERAL_FILENAME_JITTER_ALLOCATION);
+ int minAllocation = context.getConfiguration().getCount(MIN_PROP);
+ int maxAllocation = context.getConfiguration().getCount(MAX_PROP);
- if (baseAllocation <= 0) {
- log.warn("{} was set to {}, must be greater than 0. Using the default
{}.",
- Property.GENERAL_FILENAME_BASE_ALLOCATION.getKey(), baseAllocation,
- DEFAULT_BASE_ALLOCATION);
- baseAllocation = DEFAULT_BASE_ALLOCATION;
+ if (minAllocation <= 0) {
+ log.warn("{} was set to {}, but must be greater than 0. Using the
default ({}).",
+ MIN_PROP.getKey(), minAllocation, DEFAULT_MIN);
+ minAllocation = DEFAULT_MIN;
}
- int totalAllocation = baseAllocation;
- if (jitterAllocation > 0) {
- totalAllocation += RANDOM.get().nextInt(jitterAllocation);
+ if (maxAllocation < minAllocation) {
+ log.warn("{} was set to {}, must be greater than or equal to {} ({}).
Using {}.",
+
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 5bec1c33af6c464ce75f25ade7b4bd4dba0709ea
Merge: 59897142d4 4c10226c76
Author: Keith Turner
AuthorDate: Mon Aug 28 17:38:53 2023 -0400
Merge branch '2.1'
.../org/apache/accumulo/core/conf/Property.java| 11 +++-
.../server/tablets/UniqueNameAllocator.java| 33 +-
2 files changed, 42 insertions(+), 2 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index e64147f02e,77df51c5d8..1261d83b35
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -1438,8 -1834,13 +1445,10 @@@ public enum Property
return key.startsWith(Property.TABLE_PREFIX.getKey())
|| key.startsWith(Property.TSERV_PREFIX.getKey())
|| key.startsWith(Property.MANAGER_PREFIX.getKey())
-|| key.startsWith(Property.MASTER_PREFIX.getKey())
|| key.startsWith(Property.GC_PREFIX.getKey())
- || key.startsWith(Property.GENERAL_ARBITRARY_PROP_PREFIX.getKey());
+ || key.startsWith(Property.GENERAL_ARBITRARY_PROP_PREFIX.getKey())
+ || key.equals(Property.GENERAL_FILENAME_BASE_ALLOCATION.getKey())
-|| key.equals(Property.GENERAL_FILENAME_JITTER_ALLOCATION.getKey())
-|| key.startsWith(VFS_CONTEXT_CLASSPATH_PROPERTY.getKey())
-|| key.startsWith(REPLICATION_PREFIX.getKey());
++|| key.equals(Property.GENERAL_FILENAME_JITTER_ALLOCATION.getKey());
}
/**
diff --cc
server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
index 042cc5b3f0,7c0f29c5ea..5e4cc77a7d
---
a/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
@@@ -19,11 -19,15 +19,14 @@@
package org.apache.accumulo.server.tablets;
import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.security.SecureRandom;
+import static org.apache.accumulo.core.util.LazySingletons.RANDOM;
import org.apache.accumulo.core.Constants;
+ import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.util.FastFormat;
import org.apache.accumulo.server.ServerContext;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
/**
* Allocates unique names for an accumulo instance. The names are unique for
the lifetime of the
@@@ -65,4 -75,27 +73,27 @@@ public class UniqueNameAllocator
return new String(FastFormat.toZeroPaddedString(next++, 7,
Character.MAX_RADIX, new byte[0]),
UTF_8);
}
+
+ private int getAllocation() {
+ int baseAllocation =
+
context.getConfiguration().getCount(Property.GENERAL_FILENAME_BASE_ALLOCATION);
+ int jitterAllocation =
+
context.getConfiguration().getCount(Property.GENERAL_FILENAME_JITTER_ALLOCATION);
+
+ if (baseAllocation <= 0) {
+ log.warn("{} was set to {}, must be greater than 0. Using the default
{}.",
+ Property.GENERAL_FILENAME_BASE_ALLOCATION.getKey(), baseAllocation,
+ DEFAULT_BASE_ALLOCATION);
+ baseAllocation = DEFAULT_BASE_ALLOCATION;
+ }
+
+ int totalAllocation = baseAllocation;
+ if (jitterAllocation > 0) {
- totalAllocation += random.nextInt(jitterAllocation);
++ totalAllocation += RANDOM.get().nextInt(jitterAllocation);
+ }
+
+ log.debug("Allocating {} filenames", totalAllocation);
+
+ return totalAllocation;
+ }
}
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. jmark99 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 59897142d4ba39e563f1a9ad14ef512efb842abe Merge: f88500dfd0 fc793ec9a1 Author: Mark Owens AuthorDate: Mon Aug 28 10:04:22 2023 -0400 Merge branch '2.1' shell/src/main/java/org/apache/accumulo/shell/Shell.java | 16 +--- 1 file changed, 13 insertions(+), 3 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit af972f223b7cb64bb522d5706fbbf4857a19e167 Merge: cd7642988b fa796c64cd Author: Christopher Tubbs AuthorDate: Thu Aug 24 15:32:06 2023 -0400 Merge branch '2.1' .../org/apache/accumulo/core/fate/zookeeper/ZooReader.java | 10 -- .../accumulo/core/fate/zookeeper/ZooReaderWriterTest.java | 3 +++ 2 files changed, 11 insertions(+), 2 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit cd7642988bac831292e988326aaa96e07938c641 Merge: 8cd906ea18 fcaf4d35a6 Author: Christopher Tubbs AuthorDate: Wed Aug 23 12:38:05 2023 -0400 Merge branch '2.1' .../accumulo/tserver/tablet/DatafileManager.java | 19 +++ 1 file changed, 19 insertions(+)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 80e63feaaaefee58f67436866fd8b6086f96ad8f Merge: 05b0604f59 13ce0d1f27 Author: Christopher Tubbs AuthorDate: Mon Aug 21 17:29:07 2023 -0400 Merge branch '2.1' .../accumulo/server/fs/VolumeManagerImpl.java | 43 -- .../accumulo/server/fs/VolumeManagerImplTest.java | 39 2 files changed, 71 insertions(+), 11 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit a658fa8cbace3381d10555eac51c9f1fbe28b531 Merge: e6026cbdc8 ded409fc46 Author: Christopher Tubbs AuthorDate: Thu Aug 17 19:27:39 2023 -0400 Merge branch '2.1' minicluster/minicluster.properties | 31 -- pom.xml| 8 +-- .../accumulo/shell/format/DeleterFormatter.java| 56 +++--- .../shell/format/DeleterFormatterTest.java | 69 +++--- .../build/create-release-candidate.sh | 10 ++-- 5 files changed, 81 insertions(+), 93 deletions(-) diff --cc pom.xml index a2eecf76d7,033dc81abc..16a6426583 --- a/pom.xml +++ b/pom.xml @@@ -1163,7 -1174,7 +1163,7 @@@ warning true - **/thrift/*.java,**/HelpMojo.java - **/thrift/*.java,**/proto/*.java ++ **/thrift/*.java
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit e6026cbdc81bc0fec7b4c5f90459cb68336293c8 Merge: 57903f4636 64916fb724 Author: Christopher Tubbs AuthorDate: Wed Aug 16 17:05:55 2023 -0400 Merge branch '2.1' assemble/bin/accumulo | 2 +- assemble/conf/accumulo-env.sh | 9 +++-- 2 files changed, 8 insertions(+), 3 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
ctubbsii pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 57903f4636197202489f057fdbd0e2bb8f99a2f1
Merge: c99f708127 38c261d123
Author: Christopher Tubbs
AuthorDate: Mon Aug 14 15:58:35 2023 -0400
Merge branch '2.1'
.../main/java/org/apache/accumulo/shell/Shell.java | 21 ++
.../accumulo/shell/commands/ConfigCommand.java | 32 --
.../shell/commands/DeleteNamespaceCommand.java | 14 ++
.../accumulo/shell/commands/DropUserCommand.java | 9 +-
.../accumulo/shell/commands/MergeCommand.java | 11 +++-
.../accumulo/shell/commands/TableOperation.java| 24 +++-
.../shell/commands/DropUserCommandTest.java| 11 ++--
.../org/apache/accumulo/test/shell/ShellIT.java| 10 ---
8 files changed, 58 insertions(+), 74 deletions(-)
diff --cc test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java
index 130451c853,b211d3880b..93ac1459f5
--- a/test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java
@@@ -416,10 -424,35 +418,10 @@@ public class ShellIT extends SharedMini
String expectedFew = "1 123:12345 [12345678] 123456789\t12345";
exec("scan -st", true, expected);
exec("scan -st -f 5", true, expectedFew);
-// also prove that BinaryFormatter behaves same as the default
-exec("scan -st -fm org.apache.accumulo.core.util.format.BinaryFormatter",
true, expected);
-exec("scan -st -f 5 -fm
org.apache.accumulo.core.util.format.BinaryFormatter", true,
-expectedFew);
exec("setauths -c", true);
- exec("deletetable test -f", true, "Table: [test] has been deleted");
+ exec("deletetable " + name + " -f", true, "Table: [" + name + "] has been
deleted");
}
- @Test
- public void scanDateStringFormatterTest() throws IOException {
-Shell.log.debug("Starting scan dateStringFormatter test
--");
-exec("createtable t", true);
-exec("insert r f q v -ts 0", true);
-@SuppressWarnings("deprecation")
-DateFormat dateFormat =
-new
SimpleDateFormat(org.apache.accumulo.core.util.format.DateStringFormatter.DATE_FORMAT);
-String expected = String.format("r f:q [] %s\tv", dateFormat.format(new
Date(0)));
-// historically, showing few did not pertain to ColVis or Timestamp
-String expectedNoTimestamp = "r f:q []\tv";
-exec("scan -fm org.apache.accumulo.core.util.format.DateStringFormatter
-st", true, expected);
-exec("scan -fm org.apache.accumulo.core.util.format.DateStringFormatter
-st -f 1000", true,
-expected);
-exec("scan -fm org.apache.accumulo.core.util.format.DateStringFormatter
-st -f 5", true,
-expected);
-exec("scan -fm org.apache.accumulo.core.util.format.DateStringFormatter",
true,
-expectedNoTimestamp);
-exec("deletetable t -f", true, "Table: [t] has been deleted");
- }
-
@Test
public void grepTest() throws IOException {
Shell.log.debug("Starting grep test --");
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit c99f7081272647f73339fffad01871aeba483210 Merge: 67ca64eeb5 12c0b7b018 Author: Christopher Tubbs AuthorDate: Mon Aug 14 13:40:26 2023 -0400 Merge branch '2.1' .../apache/accumulo/hadoop/its/mapreduce/AccumuloInputFormatIT.java| 2 +- test/src/main/java/org/apache/accumulo/test/functional/TabletIT.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 67ca64eeb528b10203c3bfd9ae36b281b041c82c Merge: 04262c07d7 c38c10fec1 Author: Christopher Tubbs AuthorDate: Mon Aug 14 03:42:08 2023 -0400 Merge branch '2.1' pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 04262c07d776cf9b5bab5b008adb24b2c00f42b9 Merge: 12898a36b7 313238b5e8 Author: Christopher Tubbs AuthorDate: Mon Aug 14 00:44:18 2023 -0400 Merge branch '2.1' .../accumulo/core/metrics/MetricsProducer.java | 31 +++--- .../tserver/metrics/TabletServerScanMetrics.java | 2 +- 2 files changed, 17 insertions(+), 16 deletions(-) diff --cc core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java index 9e8e6ec5d1,82db7f71b2..a2aa7d6a9a --- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java +++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java @@@ -635,18 -635,22 +635,19 @@@ public interface MetricsProducer String METRICS_MINC_PREFIX = "accumulo.tserver.compactions.minc."; String METRICS_MINC_QUEUED = METRICS_MINC_PREFIX + "queued"; String METRICS_MINC_RUNNING = METRICS_MINC_PREFIX + "running"; - - String METRICS_REPLICATION_PREFIX = "accumulo.replication."; - String METRICS_REPLICATION_QUEUE = METRICS_REPLICATION_PREFIX + "queue"; - String METRICS_REPLICATION_PENDING_FILES = METRICS_REPLICATION_PREFIX + "files.pending"; - String METRICS_REPLICATION_PEERS = METRICS_REPLICATION_PREFIX + "peers"; - String METRICS_REPLICATION_THREADS = METRICS_REPLICATION_PREFIX + "threads"; + String METRICS_MINC_PAUSED = METRICS_MINC_PREFIX + "paused"; - String METRICS_SCAN = "accumulo.tserver.scans"; - String METRICS_SCAN_OPEN_FILES = METRICS_SCAN + ".files.open"; - String METRICS_SCAN_RESULTS = METRICS_SCAN + ".result"; - String METRICS_SCAN_YIELDS = METRICS_SCAN + ".yields"; - String METRICS_SCAN_START = METRICS_SCAN + ".start"; - String METRICS_SCAN_CONTINUE = METRICS_SCAN + ".continue"; - String METRICS_SCAN_CLOSE = METRICS_SCAN + ".close"; - String METRICS_SCAN_BUSY_TIMEOUT = METRICS_SCAN + ".busy_timeout"; - String METRICS_SCAN_PAUSED_FOR_MEM = METRICS_SCAN + ".paused.for.memory"; - String METRICS_SCAN_RETURN_FOR_MEM = METRICS_SCAN + ".return.early.for.memory"; + String METRICS_SCAN_PREFIX = "accumulo.tserver.scans."; + String METRICS_SCAN_TIMES = METRICS_SCAN_PREFIX + "times"; + String METRICS_SCAN_OPEN_FILES = METRICS_SCAN_PREFIX + "files.open"; + String METRICS_SCAN_RESULTS = METRICS_SCAN_PREFIX + "result"; + String METRICS_SCAN_YIELDS = METRICS_SCAN_PREFIX + "yields"; + String METRICS_SCAN_START = METRICS_SCAN_PREFIX + "start"; + String METRICS_SCAN_CONTINUE = METRICS_SCAN_PREFIX + "continue"; + String METRICS_SCAN_CLOSE = METRICS_SCAN_PREFIX + "close"; + String METRICS_SCAN_BUSY_TIMEOUT = METRICS_SCAN_PREFIX + "busy.timeout"; ++ String METRICS_SCAN_PAUSED_FOR_MEM = METRICS_SCAN_PREFIX + ".paused.for.memory"; ++ String METRICS_SCAN_RETURN_FOR_MEM = METRICS_SCAN_PREFIX + ".return.early.for.memory"; String METRICS_TSERVER_PREFIX = "accumulo.tserver."; String METRICS_TSERVER_ENTRIES = METRICS_TSERVER_PREFIX + "entries";
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit fca29a9f85bd06f0a81d637a92bb5f4700a26894
Merge: 38d28c233c e3e4c26e65
Author: Keith Turner
AuthorDate: Fri Aug 11 17:47:23 2023 -0400
Merge branch '2.1'
.../accumulo/core/classloader/ClassLoaderUtil.java | 19 ++
.../core/conf/ConfigurationTypeHelper.java | 2 +-
.../core/spi/common/ContextClassLoaderFactory.java | 4 +-
.../accumulo/server/conf/TableConfiguration.java | 8 +
.../org/apache/accumulo/server/util/PropUtil.java | 15 +-
.../accumulo/server/util/SystemPropUtil.java | 19 +-
.../accumulo/tserver/TabletClientHandler.java | 3 +
.../accumulo/tserver/tablet/CompactableImpl.java | 7 +
.../tserver/tablet/MinorCompactionTask.java| 9 +-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 78 --
.../accumulo/shell/commands/ConfigCommand.java | 31 ++-
.../test/functional/HalfClosedTablet2IT.java | 121 +
.../test/functional/HalfClosedTabletIT.java| 282 +
.../org/apache/accumulo/test/shell/ShellIT.java| 16 +-
.../apache/accumulo/test/shell/ShellServerIT.java | 79 ++
15 files changed, 651 insertions(+), 42 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/conf/ConfigurationTypeHelper.java
index 9daadb58b4,8893ae8ce9..ef5e0b3b5e
---
a/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationTypeHelper.java
+++
b/core/src/main/java/org/apache/accumulo/core/conf/ConfigurationTypeHelper.java
@@@ -172,8 -173,8 +172,8 @@@ public class ConfigurationTypeHelper
try {
instance = getClassInstance(context, clazzName, base);
-} catch (RuntimeException | IOException | ReflectiveOperationException e)
{
+} catch (RuntimeException | ReflectiveOperationException e) {
- log.warn("Failed to load class {} in classloader context {}",
clazzName, context, e);
+ log.error("Failed to load class {} in classloader context {}",
clazzName, context, e);
}
if (instance == null && defaultInstance != null) {
diff --cc
core/src/main/java/org/apache/accumulo/core/spi/common/ContextClassLoaderFactory.java
index a742ad0a33,3d9c18683f..c0e1863802
---
a/core/src/main/java/org/apache/accumulo/core/spi/common/ContextClassLoaderFactory.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/common/ContextClassLoaderFactory.java
@@@ -55,15 -55,16 +55,17 @@@ public interface ContextClassLoaderFact
default void init(ContextClassLoaderEnvironment env) {}
/**
- * Get the class loader for the given contextName. Callers should not cache
the ClassLoader result
- * as it may change if/when the ClassLoader reloads. Implementations should
throw a
- * RuntimeException of some type (such as IllegalArgumentException) if the
provided contextName is
- * not supported or fails to be constructed.
+ * Get the class loader for the given context. Callers should not cache the
ClassLoader result as
+ * it may change if/when the ClassLoader reloads. Implementations should
throw a RuntimeException
+ * of some type (such as IllegalArgumentException) if the provided context
is not supported or
+ * fails to be constructed.
*
- * @param contextName the name of the context that represents a class
loader that is managed by
- *this factory. Currently, Accumulo will only call this method for
non-null and non-empty
+ * @param context the name of the context that represents a class loader
that is managed by this
-*factory (can be null)
++ *factory. Currently, Accumulo will only call this method for
non-null and non-empty
+*context. For empty or null context, Accumulo will use the system
classloader without
+*consulting this plugin.
- * @return the class loader for the given contextName
+ * @return the class loader for the given context
*/
- ClassLoader getClassLoader(String contextName);
+ ClassLoader getClassLoader(String context);
+
}
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java
index 349e36d43a,f190d79460..ec5db3b815
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java
@@@ -18,10 -18,7 +18,8 @@@
*/
package org.apache.accumulo.tserver.tablet;
- import java.io.IOException;
-
-import org.apache.accumulo.core.metadata.TabletFile;
+import org.apache.accumulo.core.file.FilePrefix;
+import org.apache.accumulo.core.metadata.ReferencedTabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.tserver.MinorCompactionReason;
diff --cc
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 38d28c233cb21088534719fb31580f9d322a9201 Merge: 94a1d6f2a8 26a71d0627 Author: Christopher Tubbs AuthorDate: Wed Aug 2 12:35:08 2023 -0400 Merge branch '2.1' test/pom.xml | 1 + .../apache/accumulo/harness/AccumuloITBase.java| 18 .../accumulo/test/BadDeleteMarkersCreatedIT.java | 10 + .../apache/accumulo/test/ScanConsistencyIT.java| 11 + .../accumulo/test/functional/LargeRowIT.java | 12 +- .../test/functional/ManagerAssignmentIT.java | 13 +++--- .../accumulo/test/functional/WALSunnyDayIT.java| 15 +-- .../java/org/apache/accumulo/test/util/Wait.java | 48 ++ 8 files changed, 60 insertions(+), 68 deletions(-) diff --cc test/src/main/java/org/apache/accumulo/test/BadDeleteMarkersCreatedIT.java index f5f02e839b,14beb2b723..6881bd0493 --- a/test/src/main/java/org/apache/accumulo/test/BadDeleteMarkersCreatedIT.java +++ b/test/src/main/java/org/apache/accumulo/test/BadDeleteMarkersCreatedIT.java @@@ -18,9 -18,8 +18,8 @@@ */ package org.apache.accumulo.test; -import static org.apache.accumulo.core.util.UtilWaitThread.sleepUninterruptibly; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.junit.jupiter.api.Assertions.assertNotNull; - import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import java.time.Duration; diff --cc test/src/main/java/org/apache/accumulo/test/functional/LargeRowIT.java index 3658225673,bef85c4d43..3f9f7e8f2a --- a/test/src/main/java/org/apache/accumulo/test/functional/LargeRowIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/LargeRowIT.java @@@ -19,8 -19,7 +19,7 @@@ package org.apache.accumulo.test.functional; import static java.util.Collections.singletonMap; -import static org.apache.accumulo.core.util.UtilWaitThread.sleepUninterruptibly; +import static java.util.concurrent.TimeUnit.SECONDS; - import static org.junit.jupiter.api.Assertions.assertTrue; import java.time.Duration; import java.util.Map; diff --cc test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java index a9c8904f98,b1b71177e0..e194a10772 --- a/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/ManagerAssignmentIT.java @@@ -43,9 -42,7 +43,7 @@@ import org.apache.accumulo.core.clientI import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.TableId; - import org.apache.accumulo.core.data.Value; -import org.apache.accumulo.core.fate.zookeeper.ServiceLock; +import org.apache.accumulo.core.lock.ServiceLock; import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.TabletLocationState;
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 94a1d6f2a8c48237bc28dc6de7cca8b0f3615d20 Merge: 91d81531ab a6f24d7f83 Author: Christopher Tubbs AuthorDate: Wed Aug 2 05:37:23 2023 -0400 Merge branch '2.1' .../monitor/rest/tables/TableInformation.java | 12 +++--- .../rest/tservers/TabletServerInformation.java | 12 +++--- .../apache/accumulo/monitor/resources/js/table.js | 46 ++ .../accumulo/monitor/resources/js/tservers.js | 46 ++ .../apache/accumulo/monitor/templates/tables.ftl | 30 +- 5 files changed, 133 insertions(+), 13 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 91d81531ab19f240c255cac8e19db43e4490 Merge: b217d892cd fdee413764 Author: Christopher Tubbs AuthorDate: Wed Aug 2 04:53:27 2023 -0400 Merge branch '2.1' .../java/org/apache/accumulo/server/conf/util/ZooPropEditor.java | 8 1 file changed, 8 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit b217d892cddd05577597decad52d3a5074dfa7a7 Merge: 4ec6dd2054 43df0b1db4 Author: Christopher Tubbs AuthorDate: Tue Aug 1 23:10:20 2023 -0400 Merge branch '2.1'
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 27b2113ab77afd59255b57199d24d8abae0a30fa Merge: 3a83dd4355 3f48cefc7f Author: Dave Marion AuthorDate: Tue Jul 18 12:12:25 2023 + Merge branch '2.1' .../core/file/blockfile/impl/CachableBlockFile.java | 2 +- .../java/org/apache/accumulo/core/file/rfile/RFile.java | 17 ++--- 2 files changed, 11 insertions(+), 8 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 0a3feba20f2a0d0d542748776777969ba9dcd9b1 Merge: d4846d407e 0733d3ae86 Author: Keith Turner AuthorDate: Wed Jul 12 12:59:36 2023 -0400 Merge branch '2.1' .../accumulo/tserver/session/SessionManager.java | 6 +- .../accumulo/tserver/tablet/ScanDataSource.java| 11 +++ .../org/apache/accumulo/tserver/tablet/Tablet.java | 11 +++ .../tserver/session/SessionManagerTest.java| 109 + 4 files changed, 136 insertions(+), 1 deletion(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit d4846d407e5b28482394e2c0baa16932ae35e086 Merge: ffe0018b51 9bbf42f4dc Author: Dave Marion AuthorDate: Mon Jul 10 16:13:51 2023 + Merge branch '2.1' .../hadoopImpl/mapred/AccumuloRecordWriter.java| 2 +- .../hadoopImpl/mapreduce/AccumuloRecordWriter.java | 2 +- .../manager/state/LoggingTabletStateStore.java | 6 + .../server/manager/state/MetaDataStateStore.java | 15 +- .../server/manager/state/RootTabletStateStore.java | 5 +- .../server/manager/state/TabletStateStore.java | 11 +- .../server/manager/state/ZooTabletStateStore.java | 10 +- .../manager/state/RootTabletStateStoreTest.java| 3 +- .../accumulo/manager/TabletGroupWatcher.java | 49 +- .../apache/accumulo/manager/state/TableCounts.java | 9 ++ .../apache/accumulo/manager/state/TableStats.java | 6 + .../manager/tserverOps/ShutdownTServer.java| 3 + .../org/apache/accumulo/tserver/TabletServer.java | 1 + .../accumulo/tserver/UnloadTabletHandler.java | 1 + .../accumulo/tserver/session/ScanSession.java | 4 + .../apache/accumulo/tserver/session/Session.java | 6 + .../accumulo/tserver/session/SessionManager.java | 83 ++ .../org/apache/accumulo/tserver/tablet/Tablet.java | 14 +- .../test/functional/ManagerAssignmentIT.java | 167 + test/src/main/resources/log4j2-test.properties | 3 + 20 files changed, 351 insertions(+), 49 deletions(-) diff --cc server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java index b5b16e61d4,84c62254ca..acfb363d2b --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/LoggingTabletStateStore.java @@@ -26,10 -26,10 +26,11 @@@ import java.util.concurrent.TimeUnit import org.apache.accumulo.core.logging.TabletLogger; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletLocationState; + import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; -import org.apache.accumulo.core.util.HostAndPort; import org.apache.hadoop.fs.Path; +import com.google.common.net.HostAndPort; + /** * Wraps a tablet state store and logs important events. */ diff --cc server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java index 6b31568c9e,398e67c112..a2b28e1d71 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java @@@ -28,8 -28,8 +28,9 @@@ import org.apache.accumulo.core.clientI import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletLocationState; +import org.apache.accumulo.core.metadata.TabletLocationState.BadLocationStateException; import org.apache.accumulo.core.metadata.schema.Ample; + import org.apache.accumulo.core.metadata.schema.Ample.DataLevel; import org.apache.accumulo.core.metadata.schema.Ample.ReadConsistency; import org.apache.accumulo.core.metadata.schema.Ample.TabletMutator; import org.apache.accumulo.core.metadata.schema.TabletMetadata; diff --cc server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java index 7f32b7bfda,f0f8a5de2a..75f1c5f99b --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java @@@ -18,8 -18,11 +18,11 @@@ */ package org.apache.accumulo.tserver.session; + import static java.util.concurrent.TimeUnit.MILLISECONDS; + import static java.util.concurrent.TimeUnit.MINUTES; + import static java.util.concurrent.TimeUnit.SECONDS; +import static org.apache.accumulo.core.util.LazySingletons.RANDOM; -import java.security.SecureRandom; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@@ -39,10 -45,11 +45,11 @@@ import org.apache.accumulo.core.conf.Pr import org.apache.accumulo.core.data.Column; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.dataImpl.thrift.MultiScanResult; -import org.apache.accumulo.core.tabletserver.thrift.ActiveScan; -import org.apache.accumulo.core.tabletserver.thrift.ScanState; -import org.apache.accumulo.core.tabletserver.thrift.ScanType; +import org.apache.accumulo.core.tabletscan.thrift.ActiveScan; +import org.apache.accumulo.core.tabletscan.thrift.ScanState; +import org.apache.accumulo.core.tabletscan.thrift.ScanType; import org.apache.accumulo.core.util.MapCounter; + import
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 0252a64b6b665ad98349179c80727872e7c55c42 Merge: 46eca7af96 801db455a5 Author: Christopher Tubbs AuthorDate: Fri Jun 30 16:09:52 2023 -0400 Merge branch '2.1' .../compactions/InternalCompactionExecutor.java| 11 +++ .../metrics/CompactionExecutorsMetrics.java| 34 -- 2 files changed, 22 insertions(+), 23 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 46eca7af967d160e229d221c18e49be1a2a2effd Merge: 81d404e6d4 dfbefd076b Author: Christopher L. Shannon (cshannon) AuthorDate: Fri Jun 30 09:04:54 2023 -0400 Merge branch '2.1' .../org/apache/accumulo/core/metadata/ScanServerRefTabletFile.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 81d404e6d4bd377c70c98ff9e2377d21c10e2b7e
Merge: 2d12563ce1 5174be131a
Author: Christopher L. Shannon (cshannon)
AuthorDate: Fri Jun 30 08:03:58 2023 -0400
Merge branch '2.1'
.../main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
index b20724869f,24da3132ba..614368ebcb
--- a/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/StoredTabletFile.java
@@@ -18,9 -18,8 +18,10 @@@
*/
package org.apache.accumulo.core.metadata;
+ import java.net.URI;
+import java.util.Objects;
+import org.apache.accumulo.core.data.TableId;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
@@@ -44,9 -42,8 +45,9 @@@ public class StoredTabletFile extends A
* the entry can be deleted.
*/
public StoredTabletFile(String metadataEntry) {
- super(new Path(metadataEntry));
+ super(new Path(URI.create(metadataEntry)));
this.metadataEntry = metadataEntry;
+this.referencedTabletFile = ReferencedTabletFile.of(getPath());
}
/**
