(accumulo-testing) branch main updated: Create new test that creates lots of splits (#267)
This is an automated email from the ASF dual-hosted git repository. domgarguilo pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git The following commit(s) were added to refs/heads/main by this push: new 3a4a09f Create new test that creates lots of splits (#267) 3a4a09f is described below commit 3a4a09f62a0a5d60400b82b094faf5ba970bcd4f Author: Dom G AuthorDate: Fri Jan 5 16:14:43 2024 -0500 Create new test that creates lots of splits (#267) Co-authored-by: Keith Turner --- bin/cingest| 9 +- conf/accumulo-testing.properties | 23 ++ .../org/apache/accumulo/testing/TestProps.java | 14 + .../testing/continuous/ContinuousIngest.java | 197 +++--- .../accumulo/testing/continuous/CreateTable.java | 63 +++-- .../accumulo/testing/continuous/ManySplits.java| 289 + 6 files changed, 474 insertions(+), 121 deletions(-) diff --git a/bin/cingest b/bin/cingest index 43328ea..4ef312c 100755 --- a/bin/cingest +++ b/bin/cingest @@ -28,14 +28,16 @@ Usage: cingest {-o test.=} Available applications: -createtable Creates Accumulo table for continous ingest +createtable Creates Accumulo table for continuous ingest ingestInserts data into Accumulo that will form random graph. walk Randomly walks the graph using a scanner batchwalk Randomly walks the graph using a batch scanner scan Scans the graph -verifyVerifies continous ingest test. Stop ingest before running. +verifyVerifies continuous ingest test. Stop ingest before running. moru Stresses Accumulo by reading and writing to the ingest table. Stop ingest before running. +manysplitsRepeatedly lowers the split threshold on a table to create + many splits in order to test split performance bulk Create RFiles in a Map Reduce job and calls importDirectory if successful EOF } @@ -69,6 +71,9 @@ case "$1" in moru) ci_main="${ci_package}.ContinuousMoru" ;; + manysplits) +ci_main="${ci_package}.ManySplits" +;; bulk) if [ "$#" -ne 2 ]; then echo "Usage : $0 $1 " diff --git a/conf/accumulo-testing.properties b/conf/accumulo-testing.properties index b2cbb5e..93c3227 100644 --- a/conf/accumulo-testing.properties +++ b/conf/accumulo-testing.properties @@ -140,6 +140,29 @@ test.ci.bulk.map.nodes=100 # produce a bulk import file. test.ci.bulk.reducers.max=1024 +# Splits Scaling +# --- +# The number of tables to create +test.ci.split.table.count=3 +# Minimum random row to generate +test.ci.split.ingest.row.min=0 +# Maximum random row to generate +test.ci.split.ingest.row.max=9223372036854775807 +# Maximum number of random column families to generate +test.ci.split.ingest.max.cf=32767 +# Maximum number of random column qualifiers to generate +test.ci.split.ingest.max.cq=32767 +# The number of tablets to create on each table on table creation +test.ci.split.initial.tablets=1 +# The amount of data to write to each table +test.ci.split.write.size=1000 +# The split threshold to set for each table on creation +test.ci.split.threshold=1G +# The factor to reduce the split threshold by for each iteration of the test +test.ci.split.threshold.reduction.factor=10 +# Number of rounds to run the test +test.ci.split.test.rounds=3 + ### # Garbage Collection Simulation ### diff --git a/src/main/java/org/apache/accumulo/testing/TestProps.java b/src/main/java/org/apache/accumulo/testing/TestProps.java index 8d094bf..aa8e9e6 100644 --- a/src/main/java/org/apache/accumulo/testing/TestProps.java +++ b/src/main/java/org/apache/accumulo/testing/TestProps.java @@ -41,6 +41,7 @@ public class TestProps { private static final String CI_SCANNER = CI + "scanner."; private static final String CI_VERIFY = CI + "verify."; private static final String CI_BULK = CI + "bulk."; + private static final String CI_SPLIT = CI + "split."; public static final String TERASORT = PREFIX + "terasort."; public static final String ROWHASH = PREFIX + "rowhash."; @@ -148,6 +149,19 @@ public class TestProps { public static final String CI_BULK_MAP_NODES = CI_BULK + "map.nodes"; public static final String CI_BULK_REDUCERS = CI_BULK + "reducers.max"; + /** Split **/ + public static final String CI_SPLIT_TABLE_COUNT = CI_SPLIT + "table.count"; + public static final String CI_SPLIT_INGEST_ROW_MIN = CI_SPLIT + "ingest.row.min"; + public static final String CI_SPLIT_INGEST_ROW_MAX = CI_SPLIT + "ingest.row.max"; + public static final String CI_SPLIT_INGEST_MAX_CF = CI_SPLIT + "ingest.max.cf"; + public static final String CI_SPLIT_INGEST_MAX_CQ = CI_SPLIT + "ingest.max.cq"; + public static final String CI_SPLIT_INITIAL_TABLETS = CI_SPLIT +
(accumulo) branch main updated (6dc1bcfa55 -> 810d4b10e6)
This is an automated email from the ASF dual-hosted git repository. domgarguilo pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 6dc1bcfa55 Merge branch '2.1' add df268644b7 Fix ImportConfiguration.builder().setKeepOffline() (#4115) add 59b1c170a3 Update javadoc to indicate composite view returned or not for Instance, Namespace and Table Operations (#4077) new 810d4b10e6 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/client/admin/NamespaceOperations.java | 18 +++--- .../core/client/admin/TableOperations.java | 28 -- .../accumulo/manager/FateServiceHandler.java | 7 +++--- .../tableOps/tableImport/FinishImportTable.java| 5 ++-- .../manager/tableOps/tableImport/ImportTable.java | 4 ++-- .../tableOps/tableImport/ImportedTableInfo.java| 2 +- .../org/apache/accumulo/test/ImportExportIT.java | 2 ++ 7 files changed, 35 insertions(+), 31 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. domgarguilo pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 810d4b10e6381aa2ea73ca46b2e59f4f6a7334de Merge: 6dc1bcfa55 59b1c170a3 Author: Dom G AuthorDate: Fri Jan 5 11:15:28 2024 -0500 Merge branch '2.1' .../core/client/admin/NamespaceOperations.java | 18 +++--- .../core/client/admin/TableOperations.java | 28 -- .../accumulo/manager/FateServiceHandler.java | 7 +++--- .../tableOps/tableImport/FinishImportTable.java| 5 ++-- .../manager/tableOps/tableImport/ImportTable.java | 4 ++-- .../tableOps/tableImport/ImportedTableInfo.java| 2 +- .../org/apache/accumulo/test/ImportExportIT.java | 2 ++ 7 files changed, 35 insertions(+), 31 deletions(-) diff --cc test/src/main/java/org/apache/accumulo/test/ImportExportIT.java index 923ed5d8ab,49f01c1b36..0e1242dcc1 --- a/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ImportExportIT.java @@@ -46,11 -45,10 +46,12 @@@ import org.apache.accumulo.core.client. import org.apache.accumulo.core.client.admin.ImportConfiguration; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.TableId; import org.apache.accumulo.core.data.Value; + import org.apache.accumulo.core.manager.state.tables.TableState; import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily;
(accumulo) branch 2.1 updated: Update javadoc to indicate composite view returned or not for Instance, Namespace and Table Operations (#4077)
This is an automated email from the ASF dual-hosted git repository. domgarguilo pushed a commit to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/2.1 by this push: new 59b1c170a3 Update javadoc to indicate composite view returned or not for Instance, Namespace and Table Operations (#4077) 59b1c170a3 is described below commit 59b1c170a3d33123b56708bf47e3a61ae69fa40b Author: Kevin Rathbun <43969518+kevinrr...@users.noreply.github.com> AuthorDate: Fri Jan 5 11:13:20 2024 -0500 Update javadoc to indicate composite view returned or not for Instance, Namespace and Table Operations (#4077) --- .../core/client/admin/NamespaceOperations.java | 18 +++--- .../core/client/admin/TableOperations.java | 28 -- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/NamespaceOperations.java b/core/src/main/java/org/apache/accumulo/core/client/admin/NamespaceOperations.java index b82846bce6..8638eff354 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/admin/NamespaceOperations.java +++ b/core/src/main/java/org/apache/accumulo/core/client/admin/NamespaceOperations.java @@ -184,9 +184,10 @@ public interface NamespaceOperations { throws AccumuloException, AccumuloSecurityException, NamespaceNotFoundException; /** - * Gets properties of a namespace, which are inherited by tables in this namespace. Note that - * recently changed properties may not be available immediately. Method calls - * {@link #getConfiguration(String)} and then calls .entrySet() on the map. + * Gets a merged view of the properties of a namespace with its parent configuration. These + * properties are inherited by tables in this namespace. Note that recently changed properties may + * not be available immediately. Method calls {@link #getConfiguration(String)} and then calls + * .entrySet() on the map. * * @param namespace the name of the namespace * @return all properties visible by this namespace (system and per-table properties). Note that @@ -202,9 +203,9 @@ public interface NamespaceOperations { } /** - * Gets properties of a namespace, which are inherited by tables in this namespace. Note that - * recently changed properties may not be available immediately. This new method returns a Map - * instead of an Iterable. + * Gets a merged view of the properties of a namespace with its parent configuration. These + * properties are inherited by tables in this namespace. Note that recently changed properties may + * not be available immediately. This method returns a Map instead of an Iterable. * * @param namespace the name of the namespace * @return all properties visible by this namespace (system and per-table properties). Note that @@ -218,8 +219,9 @@ public interface NamespaceOperations { throws AccumuloException, AccumuloSecurityException, NamespaceNotFoundException; /** - * Gets properties specific to this namespace. Note that recently changed properties may not be - * available immediately. This new method returns a Map instead of an Iterable. + * Gets properties specific to this namespace. Note that this does not return a merged view of the + * properties with its parent configuration. Also note that recently changed properties may not be + * available immediately. This method returns a Map instead of an Iterable. * * @param namespace the name of the namespace * @return per-table properties specific to this namespace. Note that recently changed properties diff --git a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java index 768d2b5dfd..eabafaaa37 100644 --- a/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java +++ b/core/src/main/java/org/apache/accumulo/core/client/admin/TableOperations.java @@ -553,11 +553,11 @@ public interface TableOperations { throws AccumuloException, AccumuloSecurityException; /** - * Gets properties of a table. This operation is asynchronous and eventually consistent. It is not - * guaranteed that all tablets in a table will return the same values. Within a few seconds - * without another change, all tablets in a table should be consistent. The clone table feature - * can be used if consistency is required. Method calls {@link #getConfiguration(String)} and then - * calls .entrySet() on the map. + * Gets a merged view of the properties of a table with its parent configuration. This operation + * is asynchronous and eventually consistent. It is not guaranteed that all tablets in a table + * will return the same values. Within a few seconds without another change, all tablets in a + * table should be consistent. The clone
(accumulo) branch 2.1 updated: Fix ImportConfiguration.builder().setKeepOffline() (#4115)
This is an automated email from the ASF dual-hosted git repository. domgarguilo pushed a commit to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/2.1 by this push: new df268644b7 Fix ImportConfiguration.builder().setKeepOffline() (#4115) df268644b7 is described below commit df268644b766bbf84e34e7f6fd30a609ab85cc61 Author: Dom G AuthorDate: Fri Jan 5 10:55:28 2024 -0500 Fix ImportConfiguration.builder().setKeepOffline() (#4115) * fixes a bug where `setKeepOffline` would never transition a table from the NEW state to OFFLINE --- .../main/java/org/apache/accumulo/manager/FateServiceHandler.java | 7 +++ .../accumulo/manager/tableOps/tableImport/FinishImportTable.java | 5 ++--- .../apache/accumulo/manager/tableOps/tableImport/ImportTable.java | 4 ++-- .../accumulo/manager/tableOps/tableImport/ImportedTableInfo.java | 2 +- test/src/main/java/org/apache/accumulo/test/ImportExportIT.java| 2 ++ 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java b/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java index f4d1257128..6c6d0f5a32 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java @@ -607,10 +607,9 @@ class FateServiceHandler implements FateService.Iface { goalMessage += "Import table with new name: " + tableName + " from " + exportDirs; manager.fate() -.seedTransaction( -op.toString(), opid, new TraceRepo<>(new ImportTable(c.getPrincipal(), tableName, -exportDirs, namespaceId, keepMappings, !keepOffline)), -autoCleanup, goalMessage); +.seedTransaction(op.toString(), opid, new TraceRepo<>(new ImportTable(c.getPrincipal(), +tableName, exportDirs, namespaceId, keepMappings, keepOffline)), autoCleanup, +goalMessage); break; } case TABLE_EXPORT: { diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/FinishImportTable.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/FinishImportTable.java index 0df0771ecf..e5df722043 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/FinishImportTable.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/FinishImportTable.java @@ -52,9 +52,8 @@ class FinishImportTable extends ManagerRepo { } } -if (tableInfo.onlineTable) { - env.getTableManager().transitionTableState(tableInfo.tableId, TableState.ONLINE); -} +final TableState newState = tableInfo.keepOffline ? TableState.OFFLINE : TableState.ONLINE; +env.getTableManager().transitionTableState(tableInfo.tableId, newState); Utils.unreserveNamespace(env, tableInfo.namespaceId, tid, false); Utils.unreserveTable(env, tableInfo.tableId, tid, true); diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportTable.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportTable.java index 49e67f8e83..484db2b963 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportTable.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportTable.java @@ -62,14 +62,14 @@ public class ImportTable extends ManagerRepo { private final ImportedTableInfo tableInfo; public ImportTable(String user, String tableName, Set exportDirs, NamespaceId namespaceId, - boolean keepMappings, boolean onlineTable) { + boolean keepMappings, boolean keepOffline) { tableInfo = new ImportedTableInfo(); tableInfo.tableName = tableName; tableInfo.user = user; tableInfo.namespaceId = namespaceId; tableInfo.directories = parseExportDir(exportDirs); tableInfo.keepMappings = keepMappings; -tableInfo.onlineTable = onlineTable; +tableInfo.keepOffline = keepOffline; } @Override diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportedTableInfo.java b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportedTableInfo.java index f3d84a9c22..306c7a38e5 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportedTableInfo.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/tableImport/ImportedTableInfo.java @@ -35,7 +35,7 @@ class ImportedTableInfo implements Serializable { public List directories; public String exportFile; public boolean keepMappings; - public boolean onlineTable; + public boolean keepOffline;
(accumulo) branch elasticity updated: Add filtering to Ample (#3968)
This is an automated email from the ASF dual-hosted git repository. domgarguilo pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/elasticity by this push: new af3d4f1c5f Add filtering to Ample (#3968) af3d4f1c5f is described below commit af3d4f1c5fff5e8074498b0469c2e3177d3c79cc Author: Dom G AuthorDate: Fri Jan 5 10:41:07 2024 -0500 Add filtering to Ample (#3968) - Co-authored-by: Keith Turner --- .../core/iterators/user/HasCurrentFilter.java | 44 + .../user/HasExternalCompactionsFilter.java | 45 + .../core/iterators/user/HasWalsFilter.java | 45 + .../core/iterators/user/TabletMetadataFilter.java | 43 .../core/metadata/schema/TabletsMetadata.java | 53 - .../accumulo/gc/GarbageCollectWriteAheadLogs.java | 7 +- .../coordinator/CompactionCoordinator.java | 7 +- .../coordinator/DeadCompactionDetector.java| 6 +- .../monitor/rest/tables/TablesResource.java| 15 +- .../test/functional/AmpleConditionalWriterIT.java | 218 + .../test/functional/TestTabletMetadataFilter.java | 54 + 11 files changed, 520 insertions(+), 17 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/user/HasCurrentFilter.java b/core/src/main/java/org/apache/accumulo/core/iterators/user/HasCurrentFilter.java new file mode 100644 index 00..ca58922306 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/HasCurrentFilter.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.iterators.user; + +import java.util.Set; +import java.util.function.Predicate; + +import org.apache.accumulo.core.metadata.schema.TabletMetadata; + +import com.google.common.collect.Sets; + +public class HasCurrentFilter extends TabletMetadataFilter { + + public static final Set COLUMNS = + Sets.immutableEnumSet(TabletMetadata.ColumnType.LOCATION); + + private final static Predicate HAS_CURRENT = TabletMetadata::hasCurrent; + + @Override + public Set getColumns() { +return COLUMNS; + } + + @Override + protected Predicate acceptTablet() { +return HAS_CURRENT; + } +} diff --git a/core/src/main/java/org/apache/accumulo/core/iterators/user/HasExternalCompactionsFilter.java b/core/src/main/java/org/apache/accumulo/core/iterators/user/HasExternalCompactionsFilter.java new file mode 100644 index 00..fd3b5ae5d6 --- /dev/null +++ b/core/src/main/java/org/apache/accumulo/core/iterators/user/HasExternalCompactionsFilter.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.iterators.user; + +import java.util.Set; +import java.util.function.Predicate; + +import org.apache.accumulo.core.metadata.schema.TabletMetadata; + +import com.google.common.collect.Sets; + +public class HasExternalCompactionsFilter extends TabletMetadataFilter { + + public static final Set COLUMNS = + Sets.immutableEnumSet(TabletMetadata.ColumnType.ECOMP); + + private final static Predicate HAS_EXT_COMPS = + tabletMetadata -> !tabletMetadata.getExternalCompactions().isEmpty(); + + @Override + public Set getColumns() { +return COLUMNS; + } + + @Override + protected Predicate acceptTablet() { +return
(accumulo) branch elasticity updated (0d55086316 -> f46b09a7ba)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 0d55086316 removes code added for debugging merge add f46b09a7ba Avoids reading all fate ids into memory. (#4129) No new revisions were added by this update. Summary of changes: .../accumulo/core/fate/AbstractFateStore.java | 84 +++--- .../org/apache/accumulo/core/fate/AdminUtil.java | 54 +++--- .../org/apache/accumulo/core/fate/AgeOffStore.java | 14 ++-- .../java/org/apache/accumulo/core/fate/Fate.java | 17 ++--- .../org/apache/accumulo/core/fate/FateStore.java | 3 +- .../accumulo/core/fate/ReadOnlyFateStore.java | 14 ++-- .../org/apache/accumulo/core/fate/ZooStore.java| 18 - .../accumulo/core/fate/accumulo/AccumuloStore.java | 19 +++-- .../apache/accumulo/core/logging/FateLogger.java | 10 +-- .../apache/accumulo/core/fate/AgeOffStoreTest.java | 28 +++- .../org/apache/accumulo/core/fate/TestStore.java | 9 +-- .../manager/upgrade/UpgradeCoordinator.java| 16 +++-- .../java/org/apache/accumulo/test/fate/FateIT.java | 32 ++--- .../test/fate/accumulo/AccumuloFateIT.java | 13 +--- .../fate/accumulo/AccumuloStoreReadWriteIT.java| 10 +-- .../test/fate/zookeeper/ZookeeperFateIT.java | 28 ++-- 16 files changed, 171 insertions(+), 198 deletions(-)