(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 37782d558e81115a63bc85e9ada9e6ddf51d11a9 Merge: 5fb05efd26 cdd86f18f2 Author: Daniel Roberts AuthorDate: Fri Feb 2 04:33:24 2024 + Merge branch '2.1'
(accumulo) branch main updated (5fb05efd26 -> 37782d558e)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 5fb05efd26 Merge branch '2.1' add cdd86f18f2 Better error when compaction executors are not set (#4212) new 37782d558e Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
(accumulo) branch 2.1 updated: Better error when compaction executors are not set (#4212)
This is an automated email from the ASF dual-hosted git repository.
ddanielr pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new cdd86f18f2 Better error when compaction executors are not set (#4212)
cdd86f18f2 is described below
commit cdd86f18f2cce1d00a21297bb927587cbd7747bf
Author: Daniel Roberts
AuthorDate: Thu Feb 1 23:24:53 2024 -0500
Better error when compaction executors are not set (#4212)
* Better error when compaction executors are not set
Replaces NPE with proper exception type and message for when the
Default Compaction Planner's "executors" property is set and empty
or not set at all.
---
.../spi/compaction/DefaultCompactionPlanner.java | 94 --
.../compaction/DefaultCompactionPlannerTest.java | 34
2 files changed, 84 insertions(+), 44 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
index aae0591567..5a30556f63 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/compaction/DefaultCompactionPlanner.java
@@ -167,62 +167,68 @@ public class DefaultCompactionPlanner implements
CompactionPlanner {
justification = "Field is written by Gson")
@Override
public void init(InitParameters params) {
-ExecutorConfig[] execConfigs =
-new Gson().fromJson(params.getOptions().get("executors"),
ExecutorConfig[].class);
-List tmpExec = new ArrayList<>();
+if (params.getOptions().containsKey("executors")
+&& !params.getOptions().get("executors").isBlank()) {
-for (ExecutorConfig executorConfig : execConfigs) {
- Long maxSize = executorConfig.maxSize == null ? null
- :
ConfigurationTypeHelper.getFixedMemoryAsBytes(executorConfig.maxSize);
+ ExecutorConfig[] execConfigs =
+ new Gson().fromJson(params.getOptions().get("executors"),
ExecutorConfig[].class);
- CompactionExecutorId ceid;
+ List tmpExec = new ArrayList<>();
- // If not supplied, GSON will leave type null. Default to internal
- if (executorConfig.type == null) {
-executorConfig.type = "internal";
- }
+ for (ExecutorConfig executorConfig : execConfigs) {
+Long maxSize = executorConfig.maxSize == null ? null
+:
ConfigurationTypeHelper.getFixedMemoryAsBytes(executorConfig.maxSize);
- switch (executorConfig.type) {
-case "internal":
- Preconditions.checkArgument(null == executorConfig.queue,
- "'queue' should not be specified for internal compactions");
- int numThreads = Objects.requireNonNull(executorConfig.numThreads,
- "'numThreads' must be specified for internal type");
- ceid =
params.getExecutorManager().createExecutor(executorConfig.name, numThreads);
- break;
-case "external":
- Preconditions.checkArgument(null == executorConfig.numThreads,
- "'numThreads' should not be specified for external compactions");
- String queue = Objects.requireNonNull(executorConfig.queue,
- "'queue' must be specified for external type");
- ceid = params.getExecutorManager().getExternalExecutor(queue);
- break;
-default:
- throw new IllegalArgumentException("type must be 'internal' or
'external'");
- }
- tmpExec.add(new Executor(ceid, maxSize));
-}
+CompactionExecutorId ceid;
-Collections.sort(tmpExec, Comparator.comparing(Executor::getMaxSize,
-Comparator.nullsLast(Comparator.naturalOrder(;
+// If not supplied, GSON will leave type null. Default to internal
+if (executorConfig.type == null) {
+ executorConfig.type = "internal";
+}
-executors = List.copyOf(tmpExec);
+switch (executorConfig.type) {
+ case "internal":
+Preconditions.checkArgument(null == executorConfig.queue,
+"'queue' should not be specified for internal compactions");
+int numThreads = Objects.requireNonNull(executorConfig.numThreads,
+"'numThreads' must be specified for internal type");
+ceid =
params.getExecutorManager().createExecutor(executorConfig.name, numThreads);
+break;
+ case "external":
+Preconditions.checkArgument(null == executorConfig.numThreads,
+"'numThreads' should not be specified for external
compactions");
+String queue = Objects.requireNonNull(executorConfig.queue,
+"'queue' must be specified for external type");
+ceid = params.getExecutorManager().getExternalExecutor(queue);
+
(accumulo) branch elasticity updated (4eff7353c6 -> 76e75294b8)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 4eff7353c6 Use "Tablet Availability" rather than "Hosting Goal" (#3969) add b90422cb7e set max files to fix ExternalCompaction4_IT test (#4205) add 9911fc1576 Merge remote-tracking branch 'upstream/2.1' add 1db892b00d Update export version to handle imports of files without fenced ranges (#4122) add 8bf0970c7f fixes issues related to deleting a compaction service (#4211) add 5fb05efd26 Merge branch '2.1' add 76e75294b8 Merge branch 'main' into elasticity No new revisions were added by this update. Summary of changes: core/pom.xml | 2 - .../accumulo/core/metadata/StoredTabletFile.java | 15 .../core/metadata/StoredTabletFileTest.java| 26 +++--- pom.xml| 2 + .../manager/tableOps/tableExport/ExportTable.java | 10 ++- .../tableOps/tableExport/WriteExportFiles.java | 2 +- .../manager/tableOps/tableImport/ImportTable.java | 8 +- .../tableOps/tableImport/ImportedTableInfo.java| 1 + .../tableImport/PopulateMetadataTable.java | 16 +++- .../accumulo/manager/upgrade/Upgrader11to12.java | 20 + .../manager/upgrade/Upgrader11to12Test.java| 25 +- test/pom.xml | 17 .../org/apache/accumulo/test/ImportExportIT.java | 53 + .../test/compaction/ExternalCompaction4_IT.java| 12 +-- .../accumulo/test/functional/CompactionIT.java | 88 + test/src/main/resources/v2_import_test/README.md | 78 ++ .../main/resources/v2_import_test/data/A008.rf | Bin 0 -> 229 bytes .../main/resources/v2_import_test/data/A009.rf | Bin 0 -> 229 bytes .../main/resources/v2_import_test/data/A00a.rf | Bin 0 -> 228 bytes .../main/resources/v2_import_test/data/A00b.rf | Bin 0 -> 212 bytes .../main/resources/v2_import_test/data/distcp.txt | 5 ++ .../v2_import_test/data/exportMetadata.zip | Bin 0 -> 934 bytes 22 files changed, 313 insertions(+), 67 deletions(-) copy server/base/src/test/java/org/apache/accumulo/server/conf/util/ZooPropEditorTest.java => core/src/test/java/org/apache/accumulo/core/metadata/StoredTabletFileTest.java (55%) create mode 100644 test/src/main/resources/v2_import_test/README.md create mode 100644 test/src/main/resources/v2_import_test/data/A008.rf create mode 100644 test/src/main/resources/v2_import_test/data/A009.rf create mode 100644 test/src/main/resources/v2_import_test/data/A00a.rf create mode 100644 test/src/main/resources/v2_import_test/data/A00b.rf create mode 100644 test/src/main/resources/v2_import_test/data/distcp.txt create mode 100644 test/src/main/resources/v2_import_test/data/exportMetadata.zip
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 5fb05efd26d0c15f633dbd4e0d9ce8daa16282a5
Merge: 1db892b00d 8bf0970c7f
Author: Keith Turner
AuthorDate: Thu Feb 1 17:16:50 2024 -0500
Merge branch '2.1'
.../tserver/compactions/CompactionManager.java | 8 +-
.../tserver/compactions/CompactionService.java | 1 +
.../accumulo/test/functional/CompactionIT.java | 88 ++
3 files changed, 93 insertions(+), 4 deletions(-)
diff --cc
test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
index 7b633e338d,93c8c1713e..6df72f17d9
--- a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
@@@ -90,8 -77,9 +90,10 @@@ import org.apache.accumulo.core.metadat
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.security.Authorizations;
+ import org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner;
+ import org.apache.accumulo.core.spi.compaction.SimpleCompactionDispatcher;
import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.minicluster.ServerType;
import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
import org.apache.accumulo.test.VerifyIngest;
import org.apache.accumulo.test.VerifyIngest.VerifyParams;
@@@ -705,200 -628,94 +707,286 @@@ public class CompactionIT extends Accum
}
}
+ @Test
+ public void testSelectNoFiles() throws Exception {
+
+// Test a compaction selector that selects no files. In this case there
is no work to,
+// so we want to ensure it does not hang
+
+try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+
+ writeFlush(c, tableName, "a");
+ writeFlush(c, tableName, "b");
+
+ CompactionConfig config = new CompactionConfig()
+ .setSelector(new
PluginConfig(EmptyCompactionSelector.class.getName(), Map.of()))
+ .setWait(true);
+ c.tableOperations().compact(tableName, config);
+
+ assertEquals(Set.of("a", "b"), getRows(c, tableName));
+}
+
+ }
+
+ @Test
+ public void testConcurrent() throws Exception {
+// two compactions without iterators or strategy should be able to run
concurrently
+
+try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
+
+ String tableName = getUniqueNames(1)[0];
+ c.tableOperations().create(tableName);
+
+ // write random data because its very unlikely it will compress
+ writeRandomValue(c, tableName, 1 << 16);
+ writeRandomValue(c, tableName, 1 << 16);
+
+ c.tableOperations().compact(tableName, new
CompactionConfig().setWait(false));
+ c.tableOperations().compact(tableName, new
CompactionConfig().setWait(true));
+
+ assertEquals(1, FunctionalTestUtils.countRFiles(c, tableName));
+
+ writeRandomValue(c, tableName, 1 << 16);
+
+ IteratorSetting iterConfig = new IteratorSetting(30,
SlowIterator.class);
+ SlowIterator.setSleepTime(iterConfig, 1000);
+
+ long t1 = System.currentTimeMillis();
+ c.tableOperations().compact(tableName,
+ new
CompactionConfig().setWait(false).setIterators(java.util.Arrays.asList(iterConfig)));
+ try {
+// this compaction should fail because previous one set iterators
+c.tableOperations().compact(tableName, new
CompactionConfig().setWait(true));
+if (System.currentTimeMillis() - t1 < 2000) {
+ fail("Expected compaction to fail because another concurrent
compaction set iterators");
+}
+ } catch (AccumuloException e) {}
+}
+ }
+
+ @Test
+ public void testGetSelectedFilesForCompaction() throws Exception {
+
+// Tests CompactionConfigurer.InputParameters.getSelectedFiles()
+
+String tableName = this.getUniqueNames(1)[0];
+// Disable GC so intermediate compaction files are not deleted
+
getCluster().getClusterControl().stopAllServers(ServerType.GARBAGE_COLLECTOR);
+
+try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ Map props = new HashMap<>();
+ props.put(Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "none");
+ // This is done to avoid system compactions - we want to do all the
compactions ourselves
+ props.put("table.compaction.dispatcher.opts.service.system",
"nonexitant");
+ NewTableConfiguration ntc = new
NewTableConfiguration().setProperties(props);
+ client.tableOperations().create(tableName, ntc);
+
+ // The following will create 4 small and 4 large
(accumulo) branch main updated (1db892b00d -> 5fb05efd26)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 1db892b00d Update export version to handle imports of files without fenced ranges (#4122) add 8bf0970c7f fixes issues related to deleting a compaction service (#4211) new 5fb05efd26 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../tserver/compactions/CompactionManager.java | 8 +- .../tserver/compactions/CompactionService.java | 1 + .../accumulo/test/functional/CompactionIT.java | 88 ++ 3 files changed, 93 insertions(+), 4 deletions(-)
(accumulo) branch 2.1 updated: fixes issues related to deleting a compaction service (#4211)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 8bf0970c7f fixes issues related to deleting a compaction service
(#4211)
8bf0970c7f is described below
commit 8bf0970c7f1ca9cf37a16150048f983f6959f1d0
Author: Keith Turner
AuthorDate: Thu Feb 1 17:01:01 2024 -0500
fixes issues related to deleting a compaction service (#4211)
The compaction manager was not stopping deleted compaction services.
This commit fixes that and adds a test that deletes a compaction
service.
---
.../tserver/compactions/CompactionManager.java | 8 +-
.../tserver/compactions/CompactionService.java | 1 +
.../accumulo/test/functional/CompactionIT.java | 89 ++
3 files changed, 94 insertions(+), 4 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionManager.java
index be872a6ed9..4a244c4785 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionManager.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionManager.java
@@ -276,13 +276,13 @@ public class CompactionManager {
}
});
-var deletedServices =
-Sets.difference(currentCfg.getPlanners().keySet(),
tmpCfg.getPlanners().keySet());
+var deletedServices = Sets.difference(services.keySet(),
tmpServices.keySet());
-for (String serviceName : deletedServices) {
- services.get(CompactionServiceId.of(serviceName)).stop();
+for (var dcsid : deletedServices) {
+ services.get(dcsid).stop();
}
+this.currentCfg = tmpCfg;
this.services = Map.copyOf(tmpServices);
HashSet activeExternalExecs = new HashSet<>();
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
index e7bc18403d..78f5fc1173 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/compactions/CompactionService.java
@@ -444,6 +444,7 @@ public class CompactionService {
public void stop() {
executors.values().forEach(CompactionExecutor::stop);
+log.debug("Stopped compaction service {}", myId);
}
int getCompactionsRunning(CType ctype) {
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
index ec044816b2..93c8c1713e 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
@@ -30,6 +30,7 @@ import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -76,6 +77,8 @@ import
org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
import org.apache.accumulo.core.metadata.schema.TabletsMetadata;
import org.apache.accumulo.core.security.Authorizations;
+import org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner;
+import org.apache.accumulo.core.spi.compaction.SimpleCompactionDispatcher;
import org.apache.accumulo.harness.AccumuloClusterHarness;
import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
import org.apache.accumulo.test.VerifyIngest;
@@ -625,6 +628,92 @@ public class CompactionIT extends AccumuloClusterHarness {
}
}
+ @Test
+ public void testDeleteCompactionService() throws Exception {
+try (AccumuloClient c =
Accumulo.newClient().from(getClientProps()).build()) {
+ var uniqueNames = getUniqueNames(2);
+ String table1 = uniqueNames[0];
+ String table2 = uniqueNames[1];
+
+ // create a compaction service named deleteme
+ c.instanceOperations().setProperty(
+ Property.TSERV_COMPACTION_SERVICE_PREFIX.getKey() +
"deleteme.planner",
+ DefaultCompactionPlanner.class.getName());
+ c.instanceOperations().setProperty(
+ Property.TSERV_COMPACTION_SERVICE_PREFIX.getKey() +
"deleteme.planner.opts.executors",
+ "[{'name':'all','type':'internal','numThreads':1}]".replaceAll("'",
"\""));
+
+ // create a compaction service named keepme
+ c.instanceOperations().setProperty(
+ Property.TSERV_COMPACTION_SERVICE_PREFIX.getKey() + "keepme.planner",
+ DefaultCompactionPlanner.class.getName());
+
(accumulo) branch elasticity updated: fixes periodic failure with CompactionIT (#4201)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new e1f655b109 fixes periodic failure with CompactionIT (#4201)
e1f655b109 is described below
commit e1f655b1095f4113270022f905ff5da996b69d67
Author: Keith Turner
AuthorDate: Thu Feb 1 09:25:42 2024 -0500
fixes periodic failure with CompactionIT (#4201)
The test CompactionIT.testConcurrentSplit() would periodically fail
because it relied on the old behavior of WaitFor. Changed the test to
use custom code that waits up to three seconds for a condition to be met
and then continues.
---
.../accumulo/test/functional/CompactionIT.java | 33 ++
1 file changed, 21 insertions(+), 12 deletions(-)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
index b734819200..8f083fd4e3 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactionIT.java
@@ -472,12 +472,17 @@ public class CompactionIT extends AccumuloClusterHarness {
splits.add(new Text(String.format("r:%04d", i)));
}
- // wait a bit for some tablets to have files selected, it possible the
compaction have
- // completed before this so do not wait long
- Wait.waitFor(
- () -> countTablets(tableName, tabletMetadata ->
tabletMetadata.getSelectedFiles() != null)
- > 0,
- 3000, 10);
+ // Wait a bit for some tablets to have files selected, it possible the
compaction have
+ // completed before this so do not wait long. Once files are selected
compactions can start.
+ // This speed bump is an attempt to increase the chance that splits and
compactions run
+ // concurrently. Wait.waitFor() is not used here because it will throw
an exception if the
+ // time limit is exceeded.
+ long startTime = System.nanoTime();
+ while (System.nanoTime() - startTime < SECONDS.toNanos(3)
+ && countTablets(tableName, tabletMetadata ->
tabletMetadata.getSelectedFiles() != null)
+ == 0) {
+Thread.sleep(10);
+ }
// add 10 more splits to the table
c.tableOperations().addSplits(tableName, splits);
@@ -487,12 +492,16 @@ public class CompactionIT extends AccumuloClusterHarness {
splits.add(new Text(String.format("r:%04d", i)));
}
- // wait a bit for some tablets to be compacted, it possible the
compaction have completed
- // before this so do not wait long
- Wait.waitFor(
- () -> countTablets(tableName, tabletMetadata ->
!tabletMetadata.getCompacted().isEmpty())
- > 0,
- 3000, 10);
+ // Wait a bit for some tablets to be compacted, it possible the
compaction have completed
+ // before this so do not wait long. Wait.waitFor() is not used here
because it will throw an
+ // exception if the time limit is exceeded. This is just a speed bump,
its ok if the condition
+ // is not met within the time limit.
+ startTime = System.nanoTime();
+ while (System.nanoTime() - startTime < SECONDS.toNanos(3)
+ && countTablets(tableName, tabletMetadata ->
!tabletMetadata.getCompacted().isEmpty())
+ == 0) {
+Thread.sleep(10);
+ }
// add 80 more splits to the table
c.tableOperations().addSplits(tableName, splits);
