[accumulo] branch main updated (59897142d4 -> 5bec1c33af)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 59897142d4 Merge branch '2.1' add 4c10226c76 Allow configurable name allocations (#3729) new 5bec1c33af Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/core/conf/Property.java| 11 +++- .../server/tablets/UniqueNameAllocator.java| 33 +- 2 files changed, 42 insertions(+), 2 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 5bec1c33af6c464ce75f25ade7b4bd4dba0709ea
Merge: 59897142d4 4c10226c76
Author: Keith Turner
AuthorDate: Mon Aug 28 17:38:53 2023 -0400
Merge branch '2.1'
.../org/apache/accumulo/core/conf/Property.java| 11 +++-
.../server/tablets/UniqueNameAllocator.java| 33 +-
2 files changed, 42 insertions(+), 2 deletions(-)
diff --cc core/src/main/java/org/apache/accumulo/core/conf/Property.java
index e64147f02e,77df51c5d8..1261d83b35
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@@ -1438,8 -1834,13 +1445,10 @@@ public enum Property
return key.startsWith(Property.TABLE_PREFIX.getKey())
|| key.startsWith(Property.TSERV_PREFIX.getKey())
|| key.startsWith(Property.MANAGER_PREFIX.getKey())
-|| key.startsWith(Property.MASTER_PREFIX.getKey())
|| key.startsWith(Property.GC_PREFIX.getKey())
- || key.startsWith(Property.GENERAL_ARBITRARY_PROP_PREFIX.getKey());
+ || key.startsWith(Property.GENERAL_ARBITRARY_PROP_PREFIX.getKey())
+ || key.equals(Property.GENERAL_FILENAME_BASE_ALLOCATION.getKey())
-|| key.equals(Property.GENERAL_FILENAME_JITTER_ALLOCATION.getKey())
-|| key.startsWith(VFS_CONTEXT_CLASSPATH_PROPERTY.getKey())
-|| key.startsWith(REPLICATION_PREFIX.getKey());
++|| key.equals(Property.GENERAL_FILENAME_JITTER_ALLOCATION.getKey());
}
/**
diff --cc
server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
index 042cc5b3f0,7c0f29c5ea..5e4cc77a7d
---
a/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
@@@ -19,11 -19,15 +19,14 @@@
package org.apache.accumulo.server.tablets;
import static java.nio.charset.StandardCharsets.UTF_8;
-
-import java.security.SecureRandom;
+import static org.apache.accumulo.core.util.LazySingletons.RANDOM;
import org.apache.accumulo.core.Constants;
+ import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.util.FastFormat;
import org.apache.accumulo.server.ServerContext;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
/**
* Allocates unique names for an accumulo instance. The names are unique for
the lifetime of the
@@@ -65,4 -75,27 +73,27 @@@ public class UniqueNameAllocator
return new String(FastFormat.toZeroPaddedString(next++, 7,
Character.MAX_RADIX, new byte[0]),
UTF_8);
}
+
+ private int getAllocation() {
+ int baseAllocation =
+
context.getConfiguration().getCount(Property.GENERAL_FILENAME_BASE_ALLOCATION);
+ int jitterAllocation =
+
context.getConfiguration().getCount(Property.GENERAL_FILENAME_JITTER_ALLOCATION);
+
+ if (baseAllocation <= 0) {
+ log.warn("{} was set to {}, must be greater than 0. Using the default
{}.",
+ Property.GENERAL_FILENAME_BASE_ALLOCATION.getKey(), baseAllocation,
+ DEFAULT_BASE_ALLOCATION);
+ baseAllocation = DEFAULT_BASE_ALLOCATION;
+ }
+
+ int totalAllocation = baseAllocation;
+ if (jitterAllocation > 0) {
- totalAllocation += random.nextInt(jitterAllocation);
++ totalAllocation += RANDOM.get().nextInt(jitterAllocation);
+ }
+
+ log.debug("Allocating {} filenames", totalAllocation);
+
+ return totalAllocation;
+ }
}
[accumulo] branch 2.1 updated: Allow configurable name allocations (#3729)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 4c10226c76 Allow configurable name allocations (#3729)
4c10226c76 is described below
commit 4c10226c76bb71d59d31b44ca4fec392202473c0
Author: jeff <1583214+jschmid...@users.noreply.github.com>
AuthorDate: Mon Aug 28 14:32:46 2023 -0700
Allow configurable name allocations (#3729)
Update the UniqueNameAllocator to support a configurable filename range
size. The default values match what was previously there so there is no
behavior change unless you override the properties.
---
.../org/apache/accumulo/core/conf/Property.java| 9 ++
.../server/tablets/UniqueNameAllocator.java| 33 +-
2 files changed, 41 insertions(+), 1 deletion(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 9c2e9532ee..77df51c5d8 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -228,6 +228,13 @@ public enum Property {
"2.1.0"),
GENERAL_RPC_TIMEOUT("general.rpc.timeout", "120s", PropertyType.TIMEDURATION,
"Time to wait on I/O for simple, short RPC calls", "1.3.5"),
+ GENERAL_FILENAME_BASE_ALLOCATION("general.filename.base.allocation", "100",
PropertyType.COUNT,
+ "The minimum number of filenames that will be allocated from Zookeeper
at a time.", "2.1.3"),
+ GENERAL_FILENAME_JITTER_ALLOCATION("general.filename.jitter.allocation",
"100",
+ PropertyType.COUNT,
+ "The size of the jitter that will be applied to the
`general.filename.base.allocation` when allocating "
+ + "filenames from Zookeeper. This will result in an allocation
between base and (base + jitter). This property is ignored when its <= 0 and
only base is used.",
+ "2.1.3"),
@Experimental
GENERAL_RPC_SERVER_TYPE("general.rpc.server.type", "", PropertyType.STRING,
"Type of Thrift server to instantiate, see "
@@ -1830,6 +1837,8 @@ public enum Property {
|| key.startsWith(Property.MASTER_PREFIX.getKey())
|| key.startsWith(Property.GC_PREFIX.getKey())
|| key.startsWith(Property.GENERAL_ARBITRARY_PROP_PREFIX.getKey())
+|| key.equals(Property.GENERAL_FILENAME_BASE_ALLOCATION.getKey())
+|| key.equals(Property.GENERAL_FILENAME_JITTER_ALLOCATION.getKey())
|| key.startsWith(VFS_CONTEXT_CLASSPATH_PROPERTY.getKey())
|| key.startsWith(REPLICATION_PREFIX.getKey());
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
b/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
index 03eb9229bf..7c0f29c5ea 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/tablets/UniqueNameAllocator.java
@@ -23,8 +23,11 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import java.security.SecureRandom;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.util.FastFormat;
import org.apache.accumulo.server.ServerContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Allocates unique names for an accumulo instance. The names are unique for
the lifetime of the
@@ -34,6 +37,11 @@ import org.apache.accumulo.server.ServerContext;
*/
public class UniqueNameAllocator {
+ private static Logger log =
LoggerFactory.getLogger(UniqueNameAllocator.class);
+
+ private static final int DEFAULT_BASE_ALLOCATION =
+
Integer.parseInt(Property.GENERAL_FILENAME_BASE_ALLOCATION.getDefaultValue());
+
private ServerContext context;
private long next = 0;
private long maxAllocated = 0;
@@ -48,7 +56,7 @@ public class UniqueNameAllocator {
public synchronized String getNextName() {
while (next >= maxAllocated) {
- final int allocate = 100 + random.nextInt(100);
+ final int allocate = getAllocation();
try {
byte[] max = context.getZooReaderWriter().mutateExisting(nextNamePath,
currentValue -> {
@@ -67,4 +75,27 @@ public class UniqueNameAllocator {
return new String(FastFormat.toZeroPaddedString(next++, 7,
Character.MAX_RADIX, new byte[0]),
UTF_8);
}
+
+ private int getAllocation() {
+int baseAllocation =
+
context.getConfiguration().getCount(Property.GENERAL_FILENAME_BASE_ALLOCATION);
+int jitterAllocation =
+
context
[accumulo] branch 2.1 updated: Logs extent and files on failure to update tablets files (#3671)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new fcaf4d35a6 Logs extent and files on failure to update tablets files
(#3671)
fcaf4d35a6 is described below
commit fcaf4d35a67089b60b87f19f7f1a32e1d74a75df
Author: Keith Turner
AuthorDate: Tue Aug 22 13:39:24 2023 -0400
Logs extent and files on failure to update tablets files (#3671)
---
.../accumulo/tserver/tablet/DatafileManager.java | 19 +++
1 file changed, 19 insertions(+)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
index 75fd9f79f8..f54ca73339 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/DatafileManager.java
@@ -282,6 +282,12 @@ class DatafileManager {
for (Entry entry : newFiles.entrySet()) {
TabletLogger.bulkImported(tablet.getExtent(), entry.getKey());
}
+} catch (Exception e) {
+ // Any exception in this code is prone to leaving the persisted tablet
metadata and the
+ // tablets in memory data structs out of sync. Log the extent and exact
files involved as this
+ // may be useful for debugging.
+ log.error("Failure adding bulk import files {} {}", tablet.getExtent(),
paths.keySet(), e);
+ throw e;
} finally {
// increment finish count after metadata update AND updating in memory
map of files
metadataUpdateCount.updateAndGet(MetadataUpdateCount::incrementFinish);
@@ -416,6 +422,12 @@ class DatafileManager {
t2 = System.currentTimeMillis();
}
+} catch (Exception e) {
+ // Any exception in this code is prone to leaving the persisted tablet
metadata and the
+ // tablets in memory data structs out of sync. Log the extent and exact
file involved as this
+ // may be useful for debugging.
+ log.error("Failure adding minor compacted file {} {}",
tablet.getExtent(), newDatafile, e);
+ throw e;
} finally {
// increment finish count after metadata update AND updating in memory
map of files
metadataUpdateCount.updateAndGet(MetadataUpdateCount::incrementFinish);
@@ -518,6 +530,13 @@ class DatafileManager {
tablet.setLastCompactionID(compactionIdToWrite);
removeFilesAfterScan(filesInUseByScans);
+} catch (Exception e) {
+ // Any exception in this code is prone to leaving the persisted tablet
metadata and the
+ // tablets in memory data structs out of sync. Log the extent and exact
files involved as this
+ // may be useful for debugging.
+ log.error("Failure updating files after major compaction {} {} {}",
tablet.getExtent(),
+ newFile, oldDatafiles, e);
+ throw e;
} finally {
// increment finish count after metadata update AND updating in memory
map of files
metadataUpdateCount.updateAndGet(MetadataUpdateCount::incrementFinish);
(accumulo) branch elasticity updated: fixes BadCompactionServiceConfigIT
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 59a4937fc9 fixes BadCompactionServiceConfigIT
59a4937fc9 is described below
commit 59a4937fc98b5b89649e49b37f23d84f069f53f1
Author: Keith Turner
AuthorDate: Thu Nov 9 16:31:43 2023 -0500
fixes BadCompactionServiceConfigIT
The test BadCompactionServiceConfigIT was merged in from main but was
not working. Rather than fix it in a merge commit, it was fixed here.
---
.../miniclusterImpl/MiniAccumuloClusterImpl.java | 6 +--
.../compaction/BadCompactionServiceConfigIT.java | 45 ++
2 files changed, 31 insertions(+), 20 deletions(-)
diff --git
a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java
b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java
index a363554d4d..14588b04a8 100644
---
a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java
+++
b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterImpl.java
@@ -679,9 +679,9 @@ public class MiniAccumuloClusterImpl implements
AccumuloCluster {
groupNames.add(id);
}
});
- } catch (IllegalArgumentException | SecurityException |
ReflectiveOperationException e) {
-throw new RuntimeException(
-"Error creating instance of " + plannerClass + " with no-arg
constructor", e);
+ } catch (Exception e) {
+log.error("For compaction service {}, failed to get compaction queues
from planner {}.",
+serviceId, plannerClass, e);
}
}
return groupNames;
diff --git
a/test/src/main/java/org/apache/accumulo/test/compaction/BadCompactionServiceConfigIT.java
b/test/src/main/java/org/apache/accumulo/test/compaction/BadCompactionServiceConfigIT.java
index 7fc9fde216..a1d1b83e4e 100644
---
a/test/src/main/java/org/apache/accumulo/test/compaction/BadCompactionServiceConfigIT.java
+++
b/test/src/main/java/org/apache/accumulo/test/compaction/BadCompactionServiceConfigIT.java
@@ -42,30 +42,38 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.Filter;
import org.apache.accumulo.core.spi.compaction.DefaultCompactionPlanner;
-import org.apache.accumulo.harness.AccumuloClusterHarness;
+import org.apache.accumulo.harness.MiniClusterConfigurationCallback;
+import org.apache.accumulo.harness.SharedMiniClusterBase;
+import org.apache.accumulo.minicluster.ServerType;
import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import com.google.common.collect.MoreCollectors;
-//ELASTICITY_TODO
-@Disabled
-public class BadCompactionServiceConfigIT extends AccumuloClusterHarness {
+public class BadCompactionServiceConfigIT extends SharedMiniClusterBase {
private static final String CSP =
Property.TSERV_COMPACTION_SERVICE_PREFIX.getKey();
- @Override
- public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration
hadoopCoreSite) {
-Map siteCfg = new HashMap<>();
-siteCfg.put(CSP + "cs1.planner", DefaultCompactionPlanner.class.getName());
-// place invalid json in the planners config
-siteCfg.put(CSP + "cs1.planner.opts.executors", "{{'name]");
-cfg.setSiteConfig(siteCfg);
+ @BeforeAll
+ public static void beforeTests() throws Exception {
+startMiniClusterWithConfig(new ClusterConfig());
+ }
+
+ static class ClusterConfig implements MiniClusterConfigurationCallback {
+
+@Override
+public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration
hadoopCoreSite) {
+ Map siteCfg = new HashMap<>();
+ siteCfg.put(CSP + "cs1.planner",
DefaultCompactionPlanner.class.getName());
+ // place invalid json in the planners config
+ siteCfg.put(CSP + "cs1.planner.opts.executors", "{{'name]");
+ cfg.setSiteConfig(siteCfg);
+}
}
public static class EverythingFilter extends Filter {
@@ -118,9 +126,13 @@ public class BadCompactionServiceConfigIT extends
AccumuloClusterHarness {
.collect(MoreCollectors.onlyElement()));
}
- var value =
- "[{'name':'small', 'type': 'internal',
'numThreads':1}]".replaceAll("'", "\"");
+ var value = "[{'name':'all', 'type': 'external',
'group':'cs1q1'}]
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 8a886ce2c949c1cefff7fa38908017526dc4316b
Merge: 8926fe7a75 a4b4f540c8
Author: Keith Turner
AuthorDate: Thu Nov 9 15:03:31 2023 -0500
Merge branch 'main' into elasticity
.../core/metadata/schema/RootTabletMetadata.java | 65 -
.../schema/UpgraderDeprecatedConstants.java| 8 +-
.../metadata/schema/RootTabletMetadataTest.java| 119
.../accumulo/server/AccumuloDataVersion.java | 4 +-
.../server/compaction/CompactionJobGenerator.java | 51 +++-
.../compaction/ProvisionalCompactionPlanner.java | 67 +
.../server/constraints/MetadataConstraints.java| 5 +-
server/manager/pom.xml | 4 +
.../accumulo/manager/upgrade/Upgrader11to12.java | 250 ++--
.../{Upgrader11to12.java => Upgrader12to13.java} | 4 +-
.../manager/upgrade/Upgrader11to12Test.java| 317 +
.../java/org/apache/accumulo/test/MetaSplitIT.java | 62
.../compaction/BadCompactionServiceConfigIT.java | 236 +++
.../test/compaction/ExternalCompaction_1_IT.java | 56
.../test/functional/FunctionalTestUtils.java | 7 +-
.../test/functional/HalfClosedTabletIT.java| 5 +
.../apache/accumulo/test/lock/ServiceLockIT.java | 14 +-
.../accumulo/test/util/FileMetadataUtil.java | 15 +
18 files changed, 1162 insertions(+), 127 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
index e1e638f7d4,8a39e34798..c5144bb9c4
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadata.java
@@@ -166,9 -205,14 +217,13 @@@ public class RootTabletMetadata
.map(qualVal -> new SimpleImmutableEntry<>(
new Key(row, famToQualVal.getKey(), qualVal.getKey(), 1),
new Value(qualVal.getValue();
-return TabletMetadata.convertRow(entries.iterator(),
-EnumSet.allOf(TabletMetadata.ColumnType.class), false);
+return entries;
}
+ public static boolean needsUpgrade(final String json) {
+ return Data.needsUpgrade(json);
+ }
+
/**
* @return a JSON representation of the root tablet's data.
*/
diff --cc
server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
index 8d60990a74,4aae44a974..a96721988e
---
a/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/AccumuloDataVersion.java
@@@ -77,10 -77,8 +77,8 @@@ public class AccumuloDataVersion
return CURRENT_VERSION;
}
- // TODO - this disables upgrades until
https://github.com/apache/accumulo/issues/3768 is done
- // public static final Set CAN_RUN = Set.of(
- // REMOVE_DEPRECATIONS_FOR_VERSION_3, METADATA_FILE_JSON_ENCODING,
CURRENT_VERSION);
- public static final Set CAN_RUN =
- Set.of(ROOT_TABLET_META_CHANGES, REMOVE_DEPRECATIONS_FOR_VERSION_3,
CURRENT_VERSION);
++ // ELASTICITY_TODO get upgrade working
+ public static final Set CAN_RUN = Set.of(CURRENT_VERSION);
/**
* Get the stored, current working version.
diff --cc
server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionJobGenerator.java
index bf9c42516a,00..409873c073
mode 100644,00..100644
---
a/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionJobGenerator.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/compaction/CompactionJobGenerator.java
@@@ -1,281 -1,0 +1,308 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.server.compaction;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
++import java.ut
(accumulo) branch elasticity updated (8926fe7a75 -> 8a886ce2c9)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a change to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from 8926fe7a75 Updates merge code to support merging many tablets (#3934)
add 51c9957466 Improve logging of failing tests (#3924)
add 2ff8b08629 Add additional file metadata test methods (#3766)
add 999f25576c Test external compaction on table with fenced files (#3932)
add 3f51b5e0ca Update MetaSplitIT to verify fenced files (#3931)
add 75ad6ff82b Clean up LogEntry class (#3939)
add 227f84fb48 3.1 upgrade process with no chop changes (#3876)
add 4b63ead6cb Fixes handling of incorrect compaction configuration (#3912)
add a4b4f540c8 Merge branch '2.1'
new 8a886ce2c9 Merge branch 'main' into elasticity
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../core/metadata/schema/RootTabletMetadata.java | 65 -
.../schema/UpgraderDeprecatedConstants.java| 8 +-
.../metadata/schema/RootTabletMetadataTest.java| 119
.../accumulo/server/AccumuloDataVersion.java | 4 +-
.../server/compaction/CompactionJobGenerator.java | 51 +++-
.../compaction/ProvisionalCompactionPlanner.java | 67 +
.../server/constraints/MetadataConstraints.java| 5 +-
server/manager/pom.xml | 4 +
.../accumulo/manager/upgrade/Upgrader11to12.java | 250 ++--
.../{Upgrader11to12.java => Upgrader12to13.java} | 4 +-
.../manager/upgrade/Upgrader11to12Test.java| 317 +
.../java/org/apache/accumulo/test/MetaSplitIT.java | 62
.../compaction/BadCompactionServiceConfigIT.java | 236 +++
.../test/compaction/ExternalCompaction_1_IT.java | 56
.../test/functional/FunctionalTestUtils.java | 7 +-
.../test/functional/HalfClosedTabletIT.java| 5 +
.../apache/accumulo/test/lock/ServiceLockIT.java | 14 +-
.../accumulo/test/util/FileMetadataUtil.java | 15 +
18 files changed, 1162 insertions(+), 127 deletions(-)
rename
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/UpgraderConstants.java
=>
core/src/main/java/org/apache/accumulo/core/metadata/schema/UpgraderDeprecatedConstants.java
(85%)
create mode 100644
core/src/test/java/org/apache/accumulo/core/metadata/schema/RootTabletMetadataTest.java
create mode 100644
server/base/src/main/java/org/apache/accumulo/server/compaction/ProvisionalCompactionPlanner.java
copy
server/manager/src/main/java/org/apache/accumulo/manager/upgrade/{Upgrader11to12.java
=> Upgrader12to13.java} (98%)
create mode 100644
server/manager/src/test/java/org/apache/accumulo/manager/upgrade/Upgrader11to12Test.java
create mode 100644
test/src/main/java/org/apache/accumulo/test/compaction/BadCompactionServiceConfigIT.java
(accumulo) branch main updated: fixes validation in MetaSplitIT (#3944)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 71f1b0fbc4 fixes validation in MetaSplitIT (#3944)
71f1b0fbc4 is described below
commit 71f1b0fbc494342b5b659ed607ca67d5339848d1
Author: Keith Turner
AuthorDate: Sat Nov 11 15:20:07 2023 -0500
fixes validation in MetaSplitIT (#3944)
As part of validating the metadata table MetaSplitIT was requiring
tablets to have a location. However locations may be temproarily absent
and that is ok. Changed the test to validate the time field was present
as it should always be there.
---
test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java
b/test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java
index 0753463639..aeee11e9e9 100644
--- a/test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/MetaSplitIT.java
@@ -189,8 +189,9 @@ public class MetaSplitIT extends AccumuloClusterHarness {
try (var tablets =
ample.readTablets().forLevel(Ample.DataLevel.USER).build()) {
for (var tablet : tablets) {
assertTrue(expectedExtents.remove(tablet.getExtent()));
+// check a few fields that should always be present in tablet metadata
assertNotNull(tablet.getDirName());
-assertNotNull(tablet.getLocation());
+assertNotNull(tablet.getTime());
}
}
[accumulo] branch main updated (4327bee71f -> b999c6f0aa)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 4327bee71f Fix Fate print command and improve ShellServerIT test (#2755) add b999c6f0aa simplifies accumulo-cluster compactor config (#2791) No new revisions were added by this update. Summary of changes: assemble/bin/accumulo-cluster | 3 --- .../core/conf/cluster/ClusterConfigParser.java | 27 -- .../core/conf/cluster/ClusterConfigParserTest.java | 8 +++ .../cluster/cluster-with-external-compactions.yaml | 7 ++ .../apache/accumulo/core/conf/cluster/cluster.yaml | 3 --- 5 files changed, 20 insertions(+), 28 deletions(-)
[accumulo] branch main updated: fixes #2667 wait for metadata write in tablet close (#3028)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 8531c62d61 fixes #2667 wait for metadata write in tablet close (#3028)
8531c62d61 is described below
commit 8531c62d611a616225a889f42be4e7f723512564
Author: Keith Turner
AuthorDate: Tue Oct 18 12:50:13 2022 +0100
fixes #2667 wait for metadata write in tablet close (#3028)
---
.../java/org/apache/accumulo/tserver/tablet/CompactableImpl.java | 8 +---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index 20b125eea5..257322e207 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@ -1528,12 +1528,14 @@ public class CompactableImpl implements Compactable {
closed = true;
- // wait while internal jobs are running, external compactions are
committing or the status of
- // chops is MARKING, but do not wait on external compactions that are
running
+ // Wait while internal jobs are running or external compactions are
committing. When
+ // chopStatus is MARKING or selectStatus is SELECTING, there may be
metadata table writes so
+ // wait on those. Do not wait on external compactions that are running.
while (runningJobs.stream()
.anyMatch(job -> !((CompactionExecutorIdImpl)
job.getExecutor()).isExternalId())
|| !externalCompactionsCommitting.isEmpty()
- || fileMgr.chopStatus == ChopSelectionStatus.MARKING) {
+ || fileMgr.chopStatus == ChopSelectionStatus.MARKING
+ || fileMgr.selectStatus == FileSelectionStatus.SELECTING) {
try {
wait(50);
} catch (InterruptedException e) {
[accumulo] branch main updated: Fixes #3031 Makes scans more tolerant of Hadoop sutdown hook. (#3032)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 4917b3032d Fixes #3031 Makes scans more tolerant of Hadoop sutdown
hook. (#3032)
4917b3032d is described below
commit 4917b3032deb097adf1573086c0c3c3f11265f0a
Author: Keith Turner
AuthorDate: Wed Oct 19 20:53:16 2022 +0100
Fixes #3031 Makes scans more tolerant of Hadoop sutdown hook. (#3032)
---
.../apache/accumulo/core/iterators/Combiner.java | 3 ++-
.../apache/accumulo/core/util/ShutdownUtil.java| 24 ++
.../main/java/org/apache/accumulo/fate/Fate.java | 15 +-
.../apache/accumulo/tserver/tablet/Scanner.java| 11 +++---
.../apache/accumulo/tserver/tablet/TabletBase.java | 11 +-
5 files changed, 45 insertions(+), 19 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
b/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
index 23bbfeade9..fedb25ca90 100644
--- a/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
+++ b/core/src/main/java/org/apache/accumulo/core/iterators/Combiner.java
@@ -21,6 +21,7 @@ package org.apache.accumulo.core.iterators;
import static java.util.concurrent.TimeUnit.HOURS;
import java.io.IOException;
+import java.io.UncheckedIOException;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
@@ -130,7 +131,7 @@ public abstract class Combiner extends WrappingIterator
implements OptionDescrib
source.next();
hasNext = _hasNext();
} catch (IOException e) {
-throw new RuntimeException(e);
+throw new UncheckedIOException(e);
}
return topValue;
}
diff --git a/core/src/main/java/org/apache/accumulo/core/util/ShutdownUtil.java
b/core/src/main/java/org/apache/accumulo/core/util/ShutdownUtil.java
index b1cd2f0afb..0fbf90fbda 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/ShutdownUtil.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/ShutdownUtil.java
@@ -18,6 +18,8 @@
*/
package org.apache.accumulo.core.util;
+import java.io.IOException;
+
public class ShutdownUtil {
/**
@@ -34,4 +36,26 @@ public class ShutdownUtil {
return false;
}
+ public static boolean isIOException(Throwable e) {
+if (e == null)
+ return false;
+
+if (e instanceof IOException)
+ return true;
+
+for (Throwable suppressed : e.getSuppressed())
+ if (isIOException(suppressed))
+return true;
+
+return isIOException(e.getCause());
+ }
+
+ /**
+ * @return true if there is a possibility that the exception was caused by
the hadoop shutdown
+ * hook closing the hadoop file system objects, otherwise false
+ */
+ public static boolean wasCausedByHadoopShutdown(Exception e) {
+return isShutdownInProgress() && isIOException(e);
+ }
+
}
diff --git a/core/src/main/java/org/apache/accumulo/fate/Fate.java
b/core/src/main/java/org/apache/accumulo/fate/Fate.java
index d45f039e28..2a98b6e15b 100644
--- a/core/src/main/java/org/apache/accumulo/fate/Fate.java
+++ b/core/src/main/java/org/apache/accumulo/fate/Fate.java
@@ -20,6 +20,7 @@ package org.apache.accumulo.fate;
import static java.util.concurrent.TimeUnit.MINUTES;
import static java.util.concurrent.TimeUnit.SECONDS;
+import static org.apache.accumulo.core.util.ShutdownUtil.isIOException;
import static org.apache.accumulo.fate.ReadOnlyTStore.TStatus.FAILED;
import static
org.apache.accumulo.fate.ReadOnlyTStore.TStatus.FAILED_IN_PROGRESS;
import static org.apache.accumulo.fate.ReadOnlyTStore.TStatus.IN_PROGRESS;
@@ -135,20 +136,6 @@ public class Fate {
}
}
-private boolean isIOException(Throwable e) {
- if (e == null)
-return false;
-
- if (e instanceof IOException)
-return true;
-
- for (Throwable suppressed : e.getSuppressed())
-if (isIOException(suppressed))
- return true;
-
- return isIOException(e.getCause());
-}
-
/**
* The Hadoop Filesystem registers a java shutdown hook that closes the
file system. This can
* cause threads to get spurious IOException. If this happens, instead of
failing a FATE
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
index 08e2f20afa..2fd917ecda 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
@@ -125,16 +125,21 @@ public class Scanner {
else
throw iie;
} catch (IOException ioe) {
- if (ShutdownUtil.isShutdownInProgress()) {
+ if (ShutdownUtil.wasCausedByHadoopShut
[accumulo] branch 2.1 updated: fixes #3046 limiting ample to single table (#3049)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new e4674f7e5e fixes #3046 limiting ample to single table (#3049)
e4674f7e5e is described below
commit e4674f7e5ee99e4f6df3890daa74f09b1d83cfba
Author: Keith Turner
AuthorDate: Wed Oct 26 18:06:19 2022 +0100
fixes #3046 limiting ample to single table (#3049)
In PR #3044 a bug was introduced that made an Ample operation that was
intended
to read a single tables metadata read multiple tables metadata. This change
fixes that bug limiting the Ample read a to a single tables metadata.
---
.../org/apache/accumulo/core/metadata/schema/TabletsMetadata.java | 7 +--
.../org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java | 3 ++-
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
index e586b1a6ea..87dd20a5a4 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
@@ -357,8 +357,11 @@ public class TabletsMetadata implements
Iterable, AutoCloseable
@Override
public Options overlapping(Text startRow, boolean startInclusive, Text
endRow) {
- var encRow = TabletsSection.encodeRow(tableId, startRow == null ? new
Text("") : startRow);
- this.range = new Range(encRow, startRow == null ? true : startInclusive,
null, true);
+ var metaStartRow =
+ TabletsSection.encodeRow(tableId, startRow == null ? new Text("") :
startRow);
+ var metaEndRow = TabletsSection.encodeRow(tableId, endRow);
+ this.range =
+ new Range(metaStartRow, startRow == null ? true : startInclusive,
metaEndRow, true);
this.endRow = endRow;
return this;
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java
index e141b57771..a9cb992b15 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/LoadFiles.java
@@ -376,7 +376,8 @@ class LoadFiles extends ManagerRepo {
}
if (cmp != 0) {
-throw new IllegalStateException("Unexpected prev end row " +
currTablet + " " + loadRange);
+throw new IllegalStateException(
+"Unexpected prev end row " + currTablet.getExtent() + " " +
loadRange);
}
// we have found the first tablet in the range, add it to the list
[accumulo] branch 2.1 updated: Prevents bulk import from hanging. (#3044)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new d0d7b585ea Prevents bulk import from hanging. (#3044)
d0d7b585ea is described below
commit d0d7b585ea7bace2f86d5168058aaf5f33eda69c
Author: Keith Turner
AuthorDate: Tue Oct 25 23:24:24 2022 +0100
Prevents bulk import from hanging. (#3044)
Fixes a bug where bulk import would hang when the first row of the file was
equal to the last row of the first tablet.
---
.../clientImpl/bulk/ConcurrentKeyExtentCache.java | 6 +++--
.../core/metadata/schema/TabletsMetadata.java | 24 ++--
.../apache/accumulo/test/functional/BulkNewIT.java | 26 ++
3 files changed, 52 insertions(+), 4 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java
index ec154e7dbe..3f5763c10b 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/bulk/ConcurrentKeyExtentCache.java
@@ -88,13 +88,15 @@ class ConcurrentKeyExtentCache implements KeyExtentCache {
@VisibleForTesting
protected Stream lookupExtents(Text row) {
-return TabletsMetadata.builder(ctx).forTable(tableId).overlapping(row,
null).checkConsistency()
-
.fetch(PREV_ROW).build().stream().limit(100).map(TabletMetadata::getExtent);
+return TabletsMetadata.builder(ctx).forTable(tableId).overlapping(row,
true, null)
+.checkConsistency().fetch(PREV_ROW).build().stream().limit(100)
+.map(TabletMetadata::getExtent);
}
@Override
public KeyExtent lookup(Text row) {
while (true) {
+
KeyExtent ke = getFromCache(row);
if (ke != null)
return ke;
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
index 36caa39e90..e586b1a6ea 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
@@ -356,12 +356,19 @@ public class TabletsMetadata implements
Iterable, AutoCloseable
}
@Override
-public Options overlapping(Text startRow, Text endRow) {
- this.range = new KeyExtent(tableId, null, startRow).toMetaRange();
+public Options overlapping(Text startRow, boolean startInclusive, Text
endRow) {
+ var encRow = TabletsSection.encodeRow(tableId, startRow == null ? new
Text("") : startRow);
+ this.range = new Range(encRow, startRow == null ? true : startInclusive,
null, true);
this.endRow = endRow;
+
return this;
}
+@Override
+public Options overlapping(Text startRow, Text endRow) {
+ return overlapping(startRow, false, endRow);
+}
+
@Override
public Options saveKeyValues() {
this.saveKeyValues = true;
@@ -465,8 +472,21 @@ public class TabletsMetadata implements
Iterable, AutoCloseable
* Limit to tablets that overlap the range {@code (startRow, endRow]}. Can
pass null
* representing -inf and +inf. The impl creates open ended ranges which
may be problematic, see
* #813.
+ *
+ *
+ * This method is equivalent to calling {@link #overlapping(Text, boolean,
Text)} as
+ * {@code overlapping(startRow, false, endRow)}
+ *
*/
Options overlapping(Text startRow, Text endRow);
+
+/**
+ * When {@code startRowInclusive} is true limits to tablets that overlap
the range
+ * {@code [startRow,endRow]}. When {@code startRowInclusive} is false
limits to tablets that
+ * overlap the range {@code (startRow, endRow]}. Can pass null for start
and end row
+ * representing -inf and +inf.
+ */
+Options overlapping(Text startRow, boolean startRowInclusive, Text endRow);
}
private static class TabletMetadataIterator implements
Iterator {
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
index 927f17993c..afa0afc677 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/BulkNewIT.java
@@ -480,6 +480,32 @@ public class BulkNewIT extends SharedMiniClusterBase {
}
}
+ /*
+ * This test imports a file where the first row of the file is equal to the
last row of the first
+ * tablet. There was a bug where this scenario would cause bulk import to
hang forever.
+ */
+ @Test
+ public void testEndOfFirstTablet() throws Exception {
+try (Accumu
[accumulo] branch main updated: fixes user compaction stuck when producing no output (#3013)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 2c37144b62 fixes user compaction stuck when producing no output (#3013)
2c37144b62 is described below
commit 2c37144b6223280602ea4ffd24916b71c10b409d
Author: Keith Turner
AuthorDate: Wed Oct 12 12:45:20 2022 +0100
fixes user compaction stuck when producing no output (#3013)
This commit fixes a bug where :
* user compactions take multiple compaction steps (because the tablet has
many files)
* the intermediate steps produce no output
When the above happened CompactableImpl and Tablet would disagree about
what files there were.
The Tablet code would ignore the empty file produced by an intermediate
compaction.
CompactableImpl would expect Tablet to know of this file. When this
happened things would hang
until a tserver was restarted.
Ran into this bug while continually running Bulk random walk test to
reproduce #2667
---
.../accumulo/server/util/ManagerMetadataUtil.java | 10 ++--
.../accumulo/tserver/tablet/CompactableImpl.java | 57 +-
.../accumulo/tserver/tablet/CompactableUtils.java | 2 +-
.../accumulo/tserver/tablet/DatafileManager.java | 19 +---
.../tablet/CompactableImplFileManagerTest.java | 2 +-
.../accumulo/test/functional/CompactionIT.java | 51 +++
6 files changed, 105 insertions(+), 36 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java
b/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java
index 4c40125d34..82b3f92ad0 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java
@@ -182,9 +182,9 @@ public class ManagerMetadataUtil {
}
public static void replaceDatafiles(ServerContext context, KeyExtent extent,
- Set datafilesToDelete, Set
scanFiles, TabletFile path,
- Long compactionId, DataFileValue size, String address, TServerInstance
lastLocation,
- ServiceLock zooLock, Optional ecid) {
+ Set datafilesToDelete, Set scanFiles,
+ Optional path, Long compactionId, DataFileValue size,
String address,
+ TServerInstance lastLocation, ServiceLock zooLock,
Optional ecid) {
context.getAmple().putGcCandidates(extent.tableId(), datafilesToDelete);
@@ -193,8 +193,8 @@ public class ManagerMetadataUtil {
datafilesToDelete.forEach(tablet::deleteFile);
scanFiles.forEach(tablet::putScan);
-if (size.getNumEntries() > 0)
- tablet.putFile(path, size);
+if (path.isPresent())
+ tablet.putFile(path.get(), size);
if (compactionId != null)
tablet.putCompactionId(compactionId);
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
index 6e3d728b21..20b125eea5 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableImpl.java
@@ -447,7 +447,8 @@ public class CompactableImpl implements Compactable {
Set candidates = Sets.difference(selectedFiles,
allCompactingFiles);
// verify that candidates are still around and fail quietly if not
if (!currFiles.containsAll(candidates)) {
- log.debug("Selected files not in all files {} {}", candidates,
currFiles);
+ log.debug("Selected files not in all files {} {} {}",
+ Sets.difference(candidates, currFiles), candidates,
currFiles);
return Set.of();
}
// must create a copy because the sets passed to Sets.difference
could change after this
@@ -575,21 +576,21 @@ public class CompactableImpl implements Compactable {
* @param newFile
* The file produced by a compaction. If the compaction failed,
this can be null.
*/
-void completed(CompactionJob job, Set jobFiles,
StoredTabletFile newFile) {
+void completed(CompactionJob job, Set jobFiles,
+Optional newFile) {
Preconditions.checkArgument(!jobFiles.isEmpty());
Preconditions.checkState(allCompactingFiles.removeAll(jobFiles));
- if (newFile != null) {
-choppedFiles.add(newFile);
+ if (newFile.isPresent()) {
+choppedFiles.add(newFile.get());
}
- if ((job.getKind() == CompactionKind.USER || job.getKind() ==
CompactionKind.SELECTOR)
- && newFile != null) {
+ if ((job.getKind() == CompactionKind.USER || job.getKin
[accumulo-testing] branch main updated: fixes bug with bulk RW file partition point creation (#236)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new 0f7201f fixes bug with bulk RW file partition point creation (#236)
0f7201f is described below
commit 0f7201fdf69dcf839ce4066dfaf4d3f4f5a3da2e
Author: Keith Turner
AuthorDate: Thu Oct 6 15:00:14 2022 +0100
fixes bug with bulk RW file partition point creation (#236)
There was code that did the following
TreeSet startRows = new TreeSet<>();
startRows.add(0);
while (startRows.size() < parts)
startRows.add(rand.nextInt(LOTS));
The above code was replaced in 7453c37 with a stream. The stream
did not fully capture the original behavior of the loop. This
change makes the stream fully capture that behavior. Need to
ensure that `parts` unique random numbers are generated including
zero (like if the random number generator returns zero it should
be properly deduplicated). The stream was not properly handling
the RNG returning duplicates or zero.
---
.../org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java | 8 +---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git
a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
index 448a7e2..54e8209 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
@@ -59,9 +59,11 @@ public class BulkPlusOne extends BulkImportTest {
log.debug("Bulk loading from {}", dir);
final int parts = env.getRandom().nextInt(10) + 1;
-TreeSet startRows = Stream.generate(() ->
env.getRandom().nextInt(LOTS))
-.limit(parts - 1).collect(Collectors.toCollection(TreeSet::new));
-startRows.add(0);
+// The set created below should always contain 0. So its very important
that zero is first in
+// concat below.
+TreeSet startRows = Stream
+.concat(Stream.of(0), Stream.generate(() ->
env.getRandom().nextInt(LOTS))).distinct()
+.limit(parts).collect(Collectors.toCollection(TreeSet::new));
List printRows = startRows.stream().map(row -> String.format(FMT,
row))
.collect(Collectors.toList());
[accumulo] branch main updated (f758cd9b5c -> da50454871)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from f758cd9b5c Update permission validation methods to throw exceptions (#3009) add da50454871 Update the new modify properties API to automatically retry on concurrent modification exception. (#2967) No new revisions were added by this update. Summary of changes: .../core/client/admin/InstanceOperations.java | 78 ++- .../core/client/admin/NamespaceOperations.java | 25 ++- .../core/client/admin/TableOperations.java | 33 ++- .../core/clientImpl/InstanceOperationsImpl.java| 48 - .../core/clientImpl/NamespaceOperationsImpl.java | 47 - .../core/clientImpl/TableOperationsImpl.java | 48 - .../java/org/apache/accumulo/fate/util/Retry.java | 15 ++ .../core/clientImpl/TableOperationsHelperTest.java | 12 +- .../accumulo/test/conf/PropStoreConfigIT.java | 227 + 9 files changed, 498 insertions(+), 35 deletions(-)
[accumulo-testing] branch main updated: fixes two bulk import RW bugs
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new 8e83d61 fixes two bulk import RW bugs
new 2851a84 Merge pull request #235 from keith-turner/fix-rw-bulk
8e83d61 is described below
commit 8e83d616de85c784cf66f3680451a58b5420a523
Author: Keith Turner
AuthorDate: Wed Oct 5 13:08:07 2022 +
fixes two bulk import RW bugs
One bug is an off by one bug introduced when changing code from a loop to a
stream. After that change the set was one larger than it used to be.
The other bug is using a scanner outside of a try with resources block that
closes the scanner.
---
.../testing/randomwalk/bulk/BulkPlusOne.java | 4 +--
.../accumulo/testing/randomwalk/bulk/Verify.java | 40 +++---
2 files changed, 22 insertions(+), 22 deletions(-)
diff --git
a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
index e13e259..448a7e2 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/BulkPlusOne.java
@@ -59,8 +59,8 @@ public class BulkPlusOne extends BulkImportTest {
log.debug("Bulk loading from {}", dir);
final int parts = env.getRandom().nextInt(10) + 1;
-TreeSet startRows = Stream.generate(() ->
env.getRandom().nextInt(LOTS)).limit(parts)
-.collect(Collectors.toCollection(TreeSet::new));
+TreeSet startRows = Stream.generate(() ->
env.getRandom().nextInt(LOTS))
+.limit(parts - 1).collect(Collectors.toCollection(TreeSet::new));
startRows.add(0);
List printRows = startRows.stream().map(row -> String.format(FMT,
row))
diff --git
a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/Verify.java
b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/Verify.java
index 5034abb..016791c 100644
--- a/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/Verify.java
+++ b/src/main/java/org/apache/accumulo/testing/randomwalk/bulk/Verify.java
@@ -77,33 +77,33 @@ public class Verify extends Test {
scanner.clearColumns();
scanner.fetchColumnFamily(BulkPlusOne.MARKER_CF);
rowIter = new RowIterator(scanner);
-}
-while (rowIter.hasNext()) {
- Iterator> row = rowIter.next();
- long prev = 0;
- Text rowText = null;
- while (row.hasNext()) {
-Entry entry = row.next();
+ while (rowIter.hasNext()) {
+Iterator> row = rowIter.next();
+long prev = 0;
+Text rowText = null;
+while (row.hasNext()) {
+ Entry entry = row.next();
-if (rowText == null)
- rowText = entry.getKey().getRow();
+ if (rowText == null)
+rowText = entry.getKey().getRow();
-long curr =
Long.parseLong(entry.getKey().getColumnQualifier().toString());
+ long curr =
Long.parseLong(entry.getKey().getColumnQualifier().toString());
-if (curr - 1 != prev)
- throw new Exception(
- "Bad marker count " + entry.getKey() + " " + entry.getValue() +
" " + prev);
+ if (curr - 1 != prev)
+throw new Exception(
+"Bad marker count " + entry.getKey() + " " + entry.getValue()
+ " " + prev);
-if (!entry.getValue().toString().equals("1"))
- throw new Exception("Bad marker value " + entry.getKey() + " " +
entry.getValue());
+ if (!entry.getValue().toString().equals("1"))
+throw new Exception("Bad marker value " + entry.getKey() + " " +
entry.getValue());
-prev = curr;
- }
+ prev = curr;
+}
- if (BulkPlusOne.counter.get() != prev) {
-throw new Exception("Row " + rowText + " does not have all markers "
-+ BulkPlusOne.counter.get() + " " + prev);
+if (BulkPlusOne.counter.get() != prev) {
+ throw new Exception("Row " + rowText + " does not have all markers "
+ + BulkPlusOne.counter.get() + " " + prev);
+}
}
}
[accumulo] branch main updated: adds tablet id to compaction configurer and selector (#3088)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new ee4370ff7e adds tablet id to compaction configurer and selector (#3088)
ee4370ff7e is described below
commit ee4370ff7e2120142901979d3aeeba3869cbd586
Author: Keith Turner
AuthorDate: Mon Dec 5 11:03:35 2022 +
adds tablet id to compaction configurer and selector (#3088)
---
.../core/client/admin/compaction/CompactionConfigurer.java | 7 +++
.../core/client/admin/compaction/CompactionSelector.java | 7 +++
.../org/apache/accumulo/tserver/tablet/CompactableUtils.java | 12
.../strategies/ConfigurableCompactionStrategyTest.java | 6 ++
4 files changed, 32 insertions(+)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactionConfigurer.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactionConfigurer.java
index 6a58704e54..6a7e48edd5 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactionConfigurer.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactionConfigurer.java
@@ -23,6 +23,7 @@ import java.util.Map;
import org.apache.accumulo.core.client.PluginEnvironment;
import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.data.TabletId;
/**
* Enables dynamically overriding of per table properties used to create the
output file for a
@@ -50,6 +51,12 @@ public interface CompactionConfigurer {
public interface InputParameters {
TableId getTableId();
+/**
+ * @return the id of the tablet being compacted
+ * @since 3.0.0
+ */
+TabletId getTabletId();
+
public Collection getInputFiles();
PluginEnvironment getEnvironment();
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactionSelector.java
b/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactionSelector.java
index d54f612c16..281372d43d 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactionSelector.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/admin/compaction/CompactionSelector.java
@@ -30,6 +30,7 @@ import
org.apache.accumulo.core.client.summary.SummarizerConfiguration;
import org.apache.accumulo.core.client.summary.Summary;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.data.TabletId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
@@ -61,6 +62,12 @@ public interface CompactionSelector {
TableId getTableId();
+/**
+ * @return the tablet id of the tablet being compacted
+ * @since 3.0.0
+ */
+TabletId getTabletId();
+
Optional> getSample(CompactableFile cf,
SamplerConfiguration sc);
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableUtils.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableUtils.java
index 84b862f990..01cc399b85 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableUtils.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/CompactableUtils.java
@@ -52,8 +52,10 @@ import org.apache.accumulo.core.conf.ConfigurationTypeHelper;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.data.TabletId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.dataImpl.TabletIdImpl;
import org.apache.accumulo.core.file.FileOperations;
import org.apache.accumulo.core.file.FileSKVIterator;
import org.apache.accumulo.core.iterators.SortedKeyValueIterator;
@@ -262,6 +264,11 @@ public class CompactableUtils {
public TableId getTableId() {
return tablet.getExtent().tableId();
}
+
+ @Override
+ public TabletId getTabletId() {
+return new TabletIdImpl(tablet.getExtent());
+ }
});
if (overrides.getOverrides().isEmpty()) {
@@ -347,6 +354,11 @@ public class CompactableUtils {
return tablet.getExtent().tableId();
}
+ @Override
+ public TabletId getTabletId() {
+return new TabletIdImpl(tablet.getExtent());
+ }
+
@Override
public Optional>
getSample(CompactableFile file,
SamplerConfiguration sc) {
diff --git
a/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/strategies/ConfigurableCompactionStrategyTest.java
b/server/tserver/src/test/java/org/apache/accumulo/tserver/compaction/stra
[accumulo-testing] branch main updated: fixes build issue by supplying Accumulo version (#257)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-testing.git
The following commit(s) were added to refs/heads/main by this push:
new 28c7ede fixes build issue by supplying Accumulo version (#257)
28c7ede is described below
commit 28c7ede5b69e55d525cbd2c177fa31436f64a506
Author: Keith Turner
AuthorDate: Sun Nov 20 17:30:07 2022 +
fixes build issue by supplying Accumulo version (#257)
Tried to run bin/build against accumulo 2.1.1-SNAPSHOT and it
failed looking for 2.1.0-SNAPSHOT resources. This was because
script was not passing the runtime Accumulo version. Modified
the script to pass the runtime version.
---
conf/env.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/conf/env.sh b/conf/env.sh
index d2d8d67..5907d7f 100644
--- a/conf/env.sh
+++ b/conf/env.sh
@@ -82,7 +82,7 @@ fi
export HADOOP_API_JAR="${at_home}"/target/dependency/hadoop-client-api.jar
export
HADOOP_RUNTIME_JAR="${at_home}"/target/dependency/hadoop-client-runtime.jar
if [[ ! -f $HADOOP_API_JAR || ! -f $HADOOP_RUNTIME_JAR ]]; then
- mvn dependency:copy-dependencies -Dmdep.stripVersion=true
-DincludeArtifactIds=hadoop-client-api,hadoop-client-runtime
-Dhadoop.version="$HADOOP_VERSION"
+ mvn dependency:copy-dependencies -Dmdep.stripVersion=true
-DincludeArtifactIds=hadoop-client-api,hadoop-client-runtime
-Dhadoop.version="$HADOOP_VERSION" -D accumulo.version="$ACCUMULO_VERSION"
fi
# Agitator
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 459a5c0d5ef97339837a75b61af817f77fc65b42 Merge: bd49cff1be 3e47a0abde Author: Keith Turner AuthorDate: Sat Jan 21 15:23:53 2023 -0500 Merge branch '2.1' .../org/apache/accumulo/tserver/ScanServer.java| 26 +- .../apache/accumulo/tserver/ScanServerTest.java| 2 +- 2 files changed, 22 insertions(+), 6 deletions(-)
[accumulo] branch main updated (bd49cff1be -> 459a5c0d5e)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from bd49cff1be Merge branch '2.1' add 3e47a0abde Fixed scan server bugs found when testing offline scans (#3164) new 459a5c0d5e Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/tserver/ScanServer.java| 26 +- .../apache/accumulo/tserver/ScanServerTest.java| 2 +- 2 files changed, 22 insertions(+), 6 deletions(-)
[accumulo] branch 2.1 updated: Fixed scan server bugs found when testing offline scans (#3164)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 3e47a0abde Fixed scan server bugs found when testing offline scans
(#3164)
3e47a0abde is described below
commit 3e47a0abde87740cf7305d203dbd3390ed51edba
Author: Keith Turner
AuthorDate: Sat Jan 21 15:23:05 2023 -0500
Fixed scan server bugs found when testing offline scans (#3164)
---
.../org/apache/accumulo/tserver/ScanServer.java| 26 +-
.../apache/accumulo/tserver/ScanServerTest.java| 2 +-
2 files changed, 22 insertions(+), 6 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
index 5a025a156e..19698afad0 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/ScanServer.java
@@ -44,6 +44,7 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.AccumuloException;
@@ -459,7 +460,7 @@ public class ScanServer extends AbstractServer
/* This constructor is called when continuing a scan */
ScanReservation(Collection files, long myReservationId) {
this.tabletsMetadata = null;
- this.failures = null;
+ this.failures = Map.of();
this.files = files;
this.myReservationId = myReservationId;
}
@@ -525,6 +526,10 @@ public class ScanServer extends AbstractServer
LOG.info("RFFS {} extent unable to load {} as AssignmentHandler
returned false",
myReservationId, extent);
failures.add(extent);
+if (!(tabletsMetadata instanceof HashMap)) {
+ // the map returned by getTabletMetadata may not be mutable
+ tabletsMetadata = new HashMap<>(tabletsMetadata);
+}
tabletsMetadata.remove(extent);
}
}
@@ -612,12 +617,15 @@ public class ScanServer extends AbstractServer
LOG.info("RFFS {} extent unable to load {} as metadata no longer
referencing files",
myReservationId, extent);
failures.add(extent);
+if (!(tabletsMetadata instanceof HashMap)) {
+ // the map returned by getTabletMetadata may not be mutable
+ tabletsMetadata = new HashMap<>(tabletsMetadata);
+}
tabletsMetadata.remove(extent);
} else {
// remove files that are still referenced
filesToReserve.removeAll(metadataAfter.getFiles());
}
-
}
// if this is not empty it means some files that we reserved are no
longer referenced by
@@ -728,9 +736,15 @@ public class ScanServer extends AbstractServer
return
Set.copyOf(session.getTabletResolver().getTablet(sss.extent).getDatafiles().keySet());
} else if (session instanceof MultiScanSession) {
var mss = (MultiScanSession) session;
- return mss.exents.stream()
- .flatMap(e ->
mss.getTabletResolver().getTablet(e).getDatafiles().keySet().stream())
- .collect(Collectors.toUnmodifiableSet());
+ return mss.exents.stream().flatMap(e -> {
+var tablet = mss.getTabletResolver().getTablet(e);
+if (tablet == null) {
+ // not all tablets passed to a multiscan are present in the metadata
table
+ return Stream.empty();
+} else {
+ return tablet.getDatafiles().keySet().stream();
+}
+ }).collect(Collectors.toUnmodifiableSet());
} else {
throw new IllegalArgumentException("Unknown session type " +
session.getClass().getName());
}
@@ -894,6 +908,7 @@ public class ScanServer extends AbstractServer
LOG.debug("continue scan: {}", scanID);
try (ScanReservation reservation = reserveFiles(scanID)) {
+ Preconditions.checkState(reservation.getFailures().isEmpty());
return delegate.continueScan(tinfo, scanID, busyTimeout);
}
}
@@ -955,6 +970,7 @@ public class ScanServer extends AbstractServer
LOG.debug("continue multi scan: {}", scanID);
try (ScanReservation reservation = reserveFiles(scanID)) {
+ Preconditions.checkState(reservation.getFailures().isEmpty());
return delegate.continueMultiScan(tinfo, scanID, busyTimeout);
}
}
diff --git
a/server/tserver/src/test/java/org/apache/accumulo/tserver/ScanServerTest.java
b/server/tserver/src/test/java/org/apache/accumulo/tserver/ScanServerTest.java
index d2fd4f5cda..fdb79e1b00 100644
---
a/server/tserver/
[accumulo] branch 2.1 updated: fixes #3045 remove stale compactions from coordinator (#3059)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 5e4e30b68b fixes #3045 remove stale compactions from coordinator
(#3059)
5e4e30b68b is described below
commit 5e4e30b68bf14b82b48442cab33d1bd58b943a89
Author: Keith Turner
AuthorDate: Wed Nov 2 16:55:55 2022 +
fixes #3045 remove stale compactions from coordinator (#3059)
---
pom.xml| 2 +-
.../coordinator/CompactionCoordinator.java | 94 +++---
.../coordinator/CompactionCoordinatorTest.java | 64 ++-
3 files changed, 131 insertions(+), 29 deletions(-)
diff --git a/pom.xml b/pom.xml
index 96833e5eb3..abc036686a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1701,7 +1701,7 @@
[17,)
---add-opens java.base/java.lang=ALL-UNNAMED --add-opens
java.base/java.util=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED
--add-opens java.base/java.net=ALL-UNNAMED --add-opens
java.management/java.lang.management=ALL-UNNAMED --add-opens
java.management/sun.management=ALL-UNNAMED --add-opens
java.base/java.security=ALL-UNNAMED --add-opens
java.base/java.lang.reflect=ALL-UNNAMED --add-opens
java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/jav [...]
+--add-opens java.base/java.lang=ALL-UNNAMED --add-opens
java.base/java.util=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED
--add-opens java.base/java.net=ALL-UNNAMED --add-opens
java.management/java.lang.management=ALL-UNNAMED --add-opens
java.management/sun.management=ALL-UNNAMED --add-opens
java.base/java.security=ALL-UNNAMED --add-opens
java.base/java.lang.reflect=ALL-UNNAMED --add-opens
java.base/java.util.concurrent=ALL-UNNAMED --add-opens java.base/jav [...]
diff --git
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
index 27e4881143..232a6066b8 100644
---
a/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
+++
b/server/compaction-coordinator/src/main/java/org/apache/accumulo/coordinator/CompactionCoordinator.java
@@ -32,6 +32,7 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import org.apache.accumulo.coordinator.QueueSummaries.PrioTserver;
import org.apache.accumulo.core.Constants;
@@ -55,7 +56,9 @@ import org.apache.accumulo.core.dataImpl.thrift.TKeyExtent;
import org.apache.accumulo.core.fate.zookeeper.ServiceLock;
import org.apache.accumulo.core.fate.zookeeper.ZooReaderWriter;
import org.apache.accumulo.core.metadata.TServerInstance;
+import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
+import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metrics.MetricsUtil;
import org.apache.accumulo.core.rpc.ThriftUtil;
import org.apache.accumulo.core.rpc.clients.ThriftClientTypes;
@@ -90,6 +93,7 @@ import org.slf4j.LoggerFactory;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
+import com.google.common.collect.Sets;
public class CompactionCoordinator extends AbstractServer
implements CompactionCoordinatorService.Iface, LiveTServerSet.Listener {
@@ -100,8 +104,14 @@ public class CompactionCoordinator extends AbstractServer
protected static final QueueSummaries QUEUE_SUMMARIES = new QueueSummaries();
- /* Map of compactionId to RunningCompactions */
- protected static final Map RUNNING =
+ /*
+ * Map of compactionId to RunningCompactions. This is an informational cache
of what external
+ * compactions may be running. Its possible it may contain external
compactions that are not
+ * actually running. It may not contain compactions that are actually
running. The metadata table
+ * is the most authoritative source of what external compactions are
currently running, but it
+ * does not have the stats that this map has.
+ */
+ protected static final Map
RUNNING_CACHE =
new ConcurrentHashMap<>();
private static final Cache COMPLETED
=
@@ -137,6 +147,7 @@ public class CompactionCoordinator extends AbstractServer
startGCLogger(schedExecutor);
printStartupMsg();
startCompactionCleaner(schedExecutor);
+startRunningCleaner(schedExecutor);
}
@Override
@@ -170,6 +181,12 @@ public class CompactionCoordinator extends AbstractServer
ThreadPools.watchNonCriticalScheduledTask(future);
}
+ protecte
[accumulo] branch 1.10 updated: Avoids holding tablet lock while doing split computations (#3249)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.10 by this push:
new d61ba8ea3a Avoids holding tablet lock while doing split computations
(#3249)
d61ba8ea3a is described below
commit d61ba8ea3a939a51996d46742d4690a39b09522f
Author: Keith Turner
AuthorDate: Fri Mar 24 19:30:24 2023 -0400
Avoids holding tablet lock while doing split computations (#3249)
---
.../org/apache/accumulo/tserver/TabletServer.java | 2 +-
.../tserver/tablet/MinorCompactionTask.java| 2 +-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 245 ++---
3 files changed, 165 insertions(+), 84 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index e141de9608..6e29beaae3 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -2081,7 +2081,7 @@ public class TabletServer extends AccumuloServerContext
implements Runnable {
// if we need to split AND compact, we need a good way
// to decide what to do
-if (tablet.needsSplit()) {
+if (tablet.needsSplit(tablet.getSplitComputations())) {
executeSplit(tablet);
continue;
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java
index 8f922d2084..3d745fc697 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/MinorCompactionTask.java
@@ -105,7 +105,7 @@ class MinorCompactionTask implements Runnable {
minorCompaction.data("size", Long.toString(this.stats.getSize()));
minorCompaction.stop();
- if (tablet.needsSplit()) {
+ if (tablet.needsSplit(tablet.getSplitComputations())) {
tablet.getTabletServer().executeSplit(tablet);
} else {
tablet.initiateMajorCompaction(MajorCompactionReason.NORMAL);
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 6c452c4d64..5352b7e91a 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -45,6 +45,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
import org.apache.accumulo.core.Constants;
@@ -1629,13 +1630,19 @@ public class Tablet implements TabletCommitter {
return true;
}
- private SplitRowSpec findSplitRow(Collection files) {
+ private SplitRowSpec findSplitRow(Optional
splitComputations) {
long maxEndRow =
tableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE);
if (!isSplitPossible()) {
return null;
}
+if (!splitComputations.isPresent()) {
+ // information needed to compute a split point is out of date or does
not exists, try again
+ // later
+ return null;
+}
+
// have seen a big row before, do not bother checking unless a minor
compaction or map file
// import has occurred.
if (sawBigRow) {
@@ -1649,17 +1656,7 @@ public class Tablet implements TabletCommitter {
}
}
-SortedMap keys = null;
-
-try {
- // we should make .25 below configurable
- keys = FileUtil.findMidPoint(getTabletServer().getFileSystem(),
- getTabletServer().getConfiguration(), extent.getPrevEndRow(),
extent.getEndRow(),
- FileUtil.toPathStrings(files), .25);
-} catch (IOException e) {
- log.error("Failed to find midpoint " + e.getMessage());
- return null;
-}
+SortedMap keys = splitComputations.get().midPoint;
if (keys.isEmpty()) {
log.info("Cannot split tablet " + extent + ", files contain no data for
tablet.");
@@ -1673,85 +1670,75 @@ public class Tablet implements TabletCommitter {
}
// check to see if one row takes up most of the tablet, in which case we
can not split
-try {
-
- Text lastRow;
- if (extent.getEndRow() == null) {
-Key lastKey = (Key)
FileUtil.findLastKey(getTabletServer().getFileSystem(),
-getTabletServer().getConfiguration(), files);
-lastRow = lastKey.ge
[accumulo] branch 1.10 updated: Do not calculate split point in Tablet.needsSplit() (#3221)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.10 by this push:
new d6daafd894 Do not calculate split point in Tablet.needsSplit() (#3221)
d6daafd894 is described below
commit d6daafd89477af3823b955627ff7727ed67a0fb0
Author: FineAndDandy
AuthorDate: Fri Mar 24 18:14:22 2023 -0400
Do not calculate split point in Tablet.needsSplit() (#3221)
---
.../java/org/apache/accumulo/tserver/tablet/Tablet.java | 15 ---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 30bac63c27..6c452c4d64 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -1614,16 +1614,25 @@ public class Tablet implements TabletCommitter {
private long timeOfLastImportWhenBigFreakinRowWasSeen = 0;
private final long splitCreationTime;
- private SplitRowSpec findSplitRow(Collection files) {
+ private boolean isSplitPossible() {
// never split the root tablet
// check if we already decided that we can never split
// check to see if we're big enough to split
long splitThreshold =
tableConfiguration.getMemoryInBytes(Property.TABLE_SPLIT_THRESHOLD);
-long maxEndRow =
tableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE);
if (extent.isRootTablet() || estimateTabletSize() <= splitThreshold) {
+ return false;
+}
+
+return true;
+ }
+
+ private SplitRowSpec findSplitRow(Collection files) {
+long maxEndRow =
tableConfiguration.getMemoryInBytes(Property.TABLE_MAX_END_ROW_SIZE);
+
+if (!isSplitPossible()) {
return null;
}
@@ -1814,7 +1823,7 @@ public class Tablet implements TabletCommitter {
public synchronized boolean needsSplit() {
if (isClosing() || isClosed())
return false;
-return findSplitRow(getDatafileManager().getFiles()) != null;
+return isSplitPossible();
}
// BEGIN PRIVATE METHODS RELATED TO MAJOR COMPACTION
[accumulo] branch 2.1 updated (491497435d -> aa474b9de5)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git from 491497435d Fix unused variable warning add d6daafd894 Do not calculate split point in Tablet.needsSplit() (#3221) add d61ba8ea3a Avoids holding tablet lock while doing split computations (#3249) new aa474b9de5 Merge branch '1.10' into 2.1 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/tserver/TabletServer.java | 2 +- .../tserver/tablet/MinorCompactionTask.java| 2 +- .../org/apache/accumulo/tserver/tablet/Tablet.java | 231 ++--- 3 files changed, 163 insertions(+), 72 deletions(-)
[accumulo] 01/01: Merge branch '1.10' into 2.1
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit aa474b9de5287a94ade590686469373c5e5c3f68
Merge: 491497435d d61ba8ea3a
Author: Keith Turner
AuthorDate: Fri Mar 24 20:44:47 2023 -0400
Merge branch '1.10' into 2.1
.../org/apache/accumulo/tserver/TabletServer.java | 2 +-
.../tserver/tablet/MinorCompactionTask.java| 2 +-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 231 ++---
3 files changed, 163 insertions(+), 72 deletions(-)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 72e14e37fe,6e29beaae3..cbca095d55
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@@ -437,102 -729,1770 +437,102 @@@ public class TabletServer extends Abstr
}
@Override
-public InitialMultiScan startMultiScan(TInfo tinfo, TCredentials
credentials,
-Map> tbatch, List tcolumns,
List ssiList,
-Map> ssio, List authorizations,
boolean waitForWrites,
-TSamplerConfiguration tSamplerConfig, long batchTimeOut, String
context)
-throws ThriftSecurityException, TSampleNotPresentException {
- // find all of the tables that need to be scanned
- final HashSet tables = new HashSet<>();
- for (TKeyExtent keyExtent : tbatch.keySet()) {
-tables.add(new String(keyExtent.getTable(), UTF_8));
- }
-
- if (tables.size() != 1)
-throw new IllegalArgumentException("Cannot batch scan over multiple
tables");
-
- // check if user has permission to the tables
- for (String tableId : tables) {
-String namespaceId;
+public void run() {
+ while (true) {
try {
- namespaceId = Tables.getNamespaceId(getInstance(), tableId);
-} catch (TableNotFoundException e1) {
- throw new ThriftSecurityException(credentials.getPrincipal(),
- SecurityErrorCode.TABLE_DOESNT_EXIST);
-}
-if (!security.canScan(credentials, tableId, namespaceId, tbatch,
tcolumns, ssiList, ssio,
-authorizations))
- throw new ThriftSecurityException(credentials.getPrincipal(),
- SecurityErrorCode.PERMISSION_DENIED);
- }
-
- try {
-if (!security.authenticatedUserHasAuthorizations(credentials,
authorizations))
- throw new ThriftSecurityException(credentials.getPrincipal(),
- SecurityErrorCode.BAD_AUTHORIZATIONS);
- } catch (ThriftSecurityException tse) {
-log.error("{} is not authorized", credentials.getPrincipal(), tse);
-throw tse;
- }
- Map> batch = Translator.translate(tbatch, new
TKeyExtentTranslator(),
- new Translator.ListTranslator<>(new TRangeTranslator()));
-
- // This is used to determine which thread pool to use
- KeyExtent threadPoolExtent = batch.keySet().iterator().next();
-
- if (waitForWrites)
-writeTracker.waitForWrites(TabletType.type(batch.keySet()));
-
- final MultiScanSession mss = new MultiScanSession(credentials,
threadPoolExtent, batch,
- ssiList, ssio, new Authorizations(authorizations),
- SamplerConfigurationImpl.fromThrift(tSamplerConfig), batchTimeOut,
context);
-
- mss.numTablets = batch.size();
- for (List ranges : batch.values()) {
-mss.numRanges += ranges.size();
- }
-
- for (TColumn tcolumn : tcolumns)
-mss.columnSet.add(new Column(tcolumn));
-
- long sid = sessionManager.createSession(mss, true);
-
- MultiScanResult result;
- try {
-result = continueMultiScan(tinfo, sid, mss);
- } catch (NoSuchScanIDException e) {
-log.error("the impossible happened", e);
-throw new RuntimeException("the impossible happened", e);
- } finally {
-sessionManager.unreserveSession(sid);
- }
-
- return new InitialMultiScan(sid, result);
-}
-
-@Override
-public MultiScanResult continueMultiScan(TInfo tinfo, long scanID)
-throws NoSuchScanIDException, TSampleNotPresentException {
-
- MultiScanSession session = (MultiScanSession)
sessionManager.reserveSession(scanID);
-
- if (session == null) {
-throw new NoSuchScanIDException();
- }
-
- try {
-return continueMultiScan(tinfo, scanID, session);
- } finally {
-sessionManager.unreserveSession(session);
- }
-}
-
-private MultiScanResult continueMultiScan(TInfo tinfo, long scanID,
MultiScanSession session)
-throws NoSuchScanIDException, TSampleNotPresentException {
-
- if (session.lookupTask == null) {
-
[accumulo] 01/01: Merge branch '1.10' into 2.1
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 1864fe50fca922fe40c459745fcce92d72493bfd Merge: aa474b9de5 8dd79a3b31 Author: Keith Turner AuthorDate: Fri Mar 24 20:59:42 2023 -0400 Merge branch '1.10' into 2.1
[accumulo] branch 1.10 updated: fixes NPE introduced in #3249
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.10 by this push:
new 8dd79a3b31 fixes NPE introduced in #3249
8dd79a3b31 is described below
commit 8dd79a3b313241fb643195d4d85ea3b05592b428
Author: Keith Turner
AuthorDate: Fri Mar 24 20:50:42 2023 -0400
fixes NPE introduced in #3249
---
.../src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java| 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 5352b7e91a..580616ee3f 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -1871,7 +1871,7 @@ public class Tablet implements TabletCommitter {
newComputation = new SplitComputations(files, midpoint, lastRow);
} catch (IOException e) {
-lastSplitComputation = null;
+lastSplitComputation.set(null);
log.error("Failed to compute split information from files " +
e.getMessage());
return Optional.absent();
} finally {
[accumulo] branch 2.1 updated (aa474b9de5 -> 1864fe50fc)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git from aa474b9de5 Merge branch '1.10' into 2.1 add 8dd79a3b31 fixes NPE introduced in #3249 new 1864fe50fc Merge branch '1.10' into 2.1 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit ecbff2e6023240f6aaf0a231aff5b88d5dc92ab2 Merge: 7d69842000 1864fe50fc Author: Keith Turner AuthorDate: Fri Mar 24 21:00:20 2023 -0400 Merge branch '2.1' .../org/apache/accumulo/tserver/TabletServer.java | 2 +- .../tserver/tablet/MinorCompactionTask.java| 2 +- .../org/apache/accumulo/tserver/tablet/Tablet.java | 231 ++--- 3 files changed, 163 insertions(+), 72 deletions(-)
[accumulo] branch main updated (7d69842000 -> ecbff2e602)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 7d69842000 Merge branch '2.1' add d6daafd894 Do not calculate split point in Tablet.needsSplit() (#3221) add d61ba8ea3a Avoids holding tablet lock while doing split computations (#3249) add aa474b9de5 Merge branch '1.10' into 2.1 add 8dd79a3b31 fixes NPE introduced in #3249 add 1864fe50fc Merge branch '1.10' into 2.1 new ecbff2e602 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/tserver/TabletServer.java | 2 +- .../tserver/tablet/MinorCompactionTask.java| 2 +- .../org/apache/accumulo/tserver/tablet/Tablet.java | 231 ++--- 3 files changed, 163 insertions(+), 72 deletions(-)
[accumulo-website] branch main updated: Update people.md
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-website.git
The following commit(s) were added to refs/heads/main by this push:
new 9d9e85bf Update people.md
9d9e85bf is described below
commit 9d9e85bfc199594826c32054fb84511a9421a126
Author: Keith Turner
AuthorDate: Tue Mar 21 09:14:04 2023 -0400
Update people.md
---
pages/people.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pages/people.md b/pages/people.md
index beb765b6..e7b2476a 100644
--- a/pages/people.md
+++ b/pages/people.md
@@ -46,7 +46,7 @@ $(function() {
| jmark99 | [Mark Owens](https://github.com/jmark99) |
| [ET][ET] |
| jtrost| Jason Trost |
[Endgame][ENDGAME] | |
| knarendran| Karthick Narendran|
[Microsoft][MICROSOFT] |[BST][BST]|
-| kturner | [Keith Turner](https://github.com/keith-turner) |
[Microsoft][MICROSOFT] | [ET][ET] |
+| kturner | [Keith Turner](https://github.com/keith-turner) |
[Wrench.io, LLC][WRENCH] | [ET][ET] |
| lstavarez | [Luis Tavarez](https://github.com/lstav) |
| [ET][ET] |
| mdrob | Mike Drob |
[Cloudera][CLOUDERA] | [ET][ET] |
| medined | David Medinets|
| |
[accumulo] branch elasticity created (now 540179d1f5)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git at 540179d1f5 Alternate time wait calculation using nanos (#3244) No new revisions were added by this update.
[accumulo] branch elasticity updated: sets tablet hosting goals at initialization (#3325)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new eedbedf133 sets tablet hosting goals at initialization (#3325)
eedbedf133 is described below
commit eedbedf133beaf9847cd0fd8c53b313b089d6e5b
Author: Keith Turner
AuthorDate: Thu Apr 20 13:55:44 2023 -0400
sets tablet hosting goals at initialization (#3325)
---
.../java/org/apache/accumulo/server/init/FileSystemInitializer.java | 4
.../java/org/apache/accumulo/server/init/ZooKeeperInitializer.java | 5 +
2 files changed, 9 insertions(+)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java
b/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java
index daed8281b8..952ac689ca 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/init/FileSystemInitializer.java
@@ -18,6 +18,7 @@
*/
package org.apache.accumulo.server.init;
+import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.HostingColumnFamily.GOAL_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN;
@@ -29,7 +30,9 @@ import java.util.Map;
import java.util.TreeMap;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.admin.TabletHostingGoal;
import org.apache.accumulo.core.client.admin.TimeType;
+import org.apache.accumulo.core.clientImpl.TabletHostingGoalUtil;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.SiteConfiguration;
import org.apache.accumulo.core.crypto.CryptoFactoryLoader;
@@ -186,6 +189,7 @@ class FileSystemInitializer {
addEntry(map, extent, TIME_COLUMN, new Value(new MetadataTime(0,
TimeType.LOGICAL).encode()));
addEntry(map, extent, PREV_ROW_COLUMN,
MetadataSchema.TabletsSection.TabletColumnFamily.encodePrevEndRow(tablet.prevEndRow));
+addEntry(map, extent, GOAL_COLUMN,
TabletHostingGoalUtil.toValue(TabletHostingGoal.ALWAYS));
for (String file : tablet.files) {
addEntry(map, extent,
new
ColumnFQ(MetadataSchema.TabletsSection.DataFileColumnFamily.NAME, new
Text(file)),
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java
b/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java
index a939d237c1..55a77cdfdc 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/init/ZooKeeperInitializer.java
@@ -23,8 +23,10 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import java.io.IOException;
import org.apache.accumulo.core.Constants;
+import org.apache.accumulo.core.client.admin.TabletHostingGoal;
import org.apache.accumulo.core.client.admin.TimeType;
import org.apache.accumulo.core.clientImpl.Namespace;
+import org.apache.accumulo.core.clientImpl.TabletHostingGoalUtil;
import org.apache.accumulo.core.data.InstanceId;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Value;
@@ -180,6 +182,9 @@ public class ZooKeeperInitializer {
MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN.put(mutation,
new Value(new MetadataTime(0, TimeType.LOGICAL).encode()));
+MetadataSchema.TabletsSection.HostingColumnFamily.GOAL_COLUMN.put(mutation,
+TabletHostingGoalUtil.toValue(TabletHostingGoal.ALWAYS));
+
RootTabletMetadata rootTabletJson = new RootTabletMetadata();
rootTabletJson.update(mutation);
[accumulo] branch elasticity updated: Rename TabletLocator to TabletCache (#3324)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new f8258479b7 Rename TabletLocator to TabletCache (#3324)
f8258479b7 is described below
commit f8258479b75a7b8c7228d32e99126b668b51cf7c
Author: Keith Turner
AuthorDate: Thu Apr 20 12:40:28 2023 -0400
Rename TabletLocator to TabletCache (#3324)
In #3292 the TabletLocator was modified to cache tablets with out
a location. This rename is in response to the more general purpose
of the client side tablet cache.
---
.../{TabletLocator.java => ClientTabletCache.java} | 98
...LocatorImpl.java => ClientTabletCacheImpl.java} | 171 +++--
.../core/clientImpl/ConditionalWriterImpl.java | 6 +-
...bletLocator.java => RootClientTabletCache.java} | 32 +--
...tLocator.java => SyncingClientTabletCache.java} | 26 +-
.../core/clientImpl/TableOperationsImpl.java | 15 +-
.../TabletServerBatchReaderIterator.java | 20 +-
.../core/clientImpl/TabletServerBatchWriter.java | 12 +-
.../accumulo/core/clientImpl/ThriftScanner.java| 23 +-
...tLocator.java => TimeoutClientTabletCache.java} | 8 +-
.../apache/accumulo/core/clientImpl/Writer.java| 12 +-
.../core/clientImpl/ZookeeperLockChecker.java | 2 +-
...iner.java => MetadataCachedTabletObtainer.java} | 36 +--
...mplTest.java => ClientTabletCacheImplTest.java} | 271 +++--
...torTest.java => RootClientTabletCacheTest.java} | 8 +-
.../hadoopImpl/mapred/AccumuloRecordReader.java| 4 +-
.../hadoopImpl/mapreduce/AccumuloRecordReader.java | 4 +-
.../mapreduce/lib/InputConfigurator.java | 10 +-
.../accumulo/server/client/BulkImporter.java | 62 ++---
.../accumulo/server/client/BulkImporterTest.java | 20 +-
...onStateTest.java => CachedTabletStateTest.java} | 2 +-
.../org/apache/accumulo/tserver/TabletServer.java | 4 +-
.../apache/accumulo/test/BatchWriterIterator.java | 4 +-
.../accumulo/test/functional/BulkFailureIT.java| 6 +-
.../test/functional/ManagerAssignmentIT.java | 28 +--
.../test/functional/OnDemandTabletUnloadingIT.java | 6 +-
.../accumulo/test/manager/SuspendedTabletsIT.java | 8 +-
27 files changed, 450 insertions(+), 448 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletLocator.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java
similarity index 76%
rename from
core/src/main/java/org/apache/accumulo/core/clientImpl/TabletLocator.java
rename to
core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java
index f15d6c1984..adbe1d2b7e 100644
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletLocator.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java
@@ -37,7 +37,7 @@ import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.metadata.MetadataLocationObtainer;
+import org.apache.accumulo.core.metadata.MetadataCachedTabletObtainer;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.singletons.SingletonManager;
@@ -48,11 +48,14 @@ import org.apache.hadoop.io.Text;
import com.google.common.base.Preconditions;
-// ELASTICITY_TODO rename to TabletCache
-public abstract class TabletLocator {
+/**
+ * Client side cache of information about Tablets. Currently, a tablet prev
end row is cached and
+ * locations are cached if they exists.
+ */
+public abstract class ClientTabletCache {
/**
- * Flipped false on call to {@link #clearLocators}. Checked by client
classes that locally cache
+ * Flipped false on call to {@link #clearInstances}. Checked by client
classes that locally cache
* Locators.
*/
private volatile boolean isValid = true;
@@ -69,7 +72,6 @@ public abstract class TabletLocator {
REQUIRED, NOT_REQUIRED
}
- // ELASTICITY_TODO rename to findTablet
/**
* Finds the tablet that contains the given row.
*
@@ -80,17 +82,17 @@ public abstract class TabletLocator {
* @return overlapping tablet. If no overlapping tablet exists, returns
null. If location is
* required and the tablet currently has no location ,returns null.
*/
- public abstract TabletLocation locateTablet(ClientContext context, Text row,
boolean skipRow,
+ public abstract CachedTablet findTablet(ClientContext context, Text row,
boolean skipRow,
LocationNeed locationNeed)
throws AccumuloException, AccumuloSecurityException,
TableNotFoundException;
- public TabletLoc
[accumulo] branch elasticity updated (f7e823933c -> 3836ba9557)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from f7e823933c Merge branch 'main' into elasticity add 3836ba9557 updates splitRangeByTablets to not require tablet location (#3330) No new revisions were added by this update. Summary of changes: .../core/clientImpl/TableOperationsImpl.java | 22 ++ 1 file changed, 10 insertions(+), 12 deletions(-)
[accumulo] branch elasticity updated: fixes #3252 support tablets w/ and w/o location in tablet location cache (#3292)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 4cfa749e00 fixes #3252 support tablets w/ and w/o location in tablet
location cache (#3292)
4cfa749e00 is described below
commit 4cfa749e00e6e15fc22fe65d2732fa8b76ea725a
Author: Keith Turner
AuthorDate: Wed Apr 19 22:12:41 2023 -0400
fixes #3252 support tablets w/ and w/o location in tablet location cache
(#3292)
Modifies the TabletLocatorCache to support tablets with and without a
location. Also modified the ScanServerSelector to support optionally
waiting for scan servers when there are none instead of always falling
back to tservers. The scan severs selector falling back to the tserver
was written with the assumption that tablets are always hosted. Now
that tablets are loaded onto tservers on demand it may not be desirable
to load the tablets when there are no scan servers. That is why this
changes also included changes to optionally wait for scan servers when
there are none. Restarting all scan servers will be a normal
administrative operations, may not want a lot of tablets to all of a
sudden load on tablet servers when the scan servers are restarted.
---
.../core/clientImpl/RootTabletLocator.java | 20 +-
.../core/clientImpl/SyncingTabletLocator.java | 9 +-
.../core/clientImpl/TableOperationsImpl.java | 69 -
.../accumulo/core/clientImpl/TabletLocator.java| 96 +--
.../core/clientImpl/TabletLocatorImpl.java | 283 ++---
.../TabletServerBatchReaderIterator.java | 55 +++-
.../accumulo/core/clientImpl/ThriftScanner.java| 103 +++-
.../core/clientImpl/TimeoutTabletLocator.java | 5 +-
.../apache/accumulo/core/clientImpl/Writer.java| 5 +-
.../core/metadata/MetadataLocationObtainer.java| 15 +-
.../spi/scan/ConfigurableScanServerSelector.java | 51 +++-
.../accumulo/core/spi/scan/ScanServerSelector.java | 39 ++-
.../core/clientImpl/TabletLocatorImplTest.java | 144 ++-
.../scan/ConfigurableScanServerSelectorTest.java | 70 -
.../accumulo/server/client/BulkImporter.java | 5 +-
.../accumulo/server/client/BulkImporterTest.java | 4 +-
.../accumulo/test/ScanServerIT_NoServers.java | 60 +
.../accumulo/test/functional/BulkFailureIT.java| 5 +-
.../test/functional/OnDemandTabletUnloadingIT.java | 108 +++-
.../accumulo/test/manager/SuspendedTabletsIT.java | 8 +-
20 files changed, 892 insertions(+), 262 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/RootTabletLocator.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/RootTabletLocator.java
index f29a27b591..08ec4c9aa0 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/RootTabletLocator.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/RootTabletLocator.java
@@ -44,6 +44,8 @@ import org.apache.hadoop.io.Text;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
+
public class RootTabletLocator extends TabletLocator {
private final TabletServerLockChecker lockChecker;
@@ -57,11 +59,11 @@ public class RootTabletLocator extends TabletLocator {
Map> binnedMutations, List failures) {
TabletLocation rootTabletLocation = getRootTabletLocation(context);
if (rootTabletLocation != null) {
- var tsm = new
TabletServerMutations(rootTabletLocation.getTserverSession());
+ var tsm = new
TabletServerMutations(rootTabletLocation.getTserverSession().get());
for (T mutation : mutations) {
tsm.addMutation(RootTable.EXTENT, mutation);
}
- binnedMutations.put(rootTabletLocation.getTserverLocation(), tsm);
+ binnedMutations.put(rootTabletLocation.getTserverLocation().get(), tsm);
} else {
failures.addAll(mutations);
}
@@ -69,7 +71,11 @@ public class RootTabletLocator extends TabletLocator {
@Override
public List locateTablets(ClientContext context, List ranges,
- BiConsumer rangeConsumer) {
+ BiConsumer rangeConsumer, LocationNeed
locationNeed) {
+
+// only expect the hosted case so this code only handles that, so throw an
exception is
+// something else is seen
+Preconditions.checkArgument(locationNeed == LocationNeed.REQUIRED);
TabletLocation rootTabletLocation = getRootTabletLocation(context);
if (rootTabletLocation != null) {
@@ -132,10 +138,14 @@ public class RootTabletLocator extends TabletLocator {
@Override
public TabletLocation locateTablet(ClientContext context, Text row, boolean
skipRow,
- boolean retry) {
+ LocationNeed locationNeed) {
+// only expect the hosted case so this code only handles that, so th
[accumulo] branch main updated: get table splits via ample (#3338)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 8eaecd3cc9 get table splits via ample (#3338)
8eaecd3cc9 is described below
commit 8eaecd3cc9b78c6a0198aee15a3505f24f8cafd8
Author: Keith Turner
AuthorDate: Tue Apr 25 14:21:49 2023 -0400
get table splits via ample (#3338)
fixes #
---
.../core/clientImpl/TableOperationsImpl.java | 35 +++---
.../core/metadata/schema/TabletsMetadata.java | 2 ++
2 files changed, 12 insertions(+), 25 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index 1da446f496..93734a39b1 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@ -119,9 +119,9 @@ import
org.apache.accumulo.core.manager.state.tables.TableState;
import org.apache.accumulo.core.manager.thrift.FateOperation;
import org.apache.accumulo.core.manager.thrift.FateService;
import org.apache.accumulo.core.manager.thrift.ManagerClientService;
-import org.apache.accumulo.core.metadata.MetadataServicer;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
+import org.apache.accumulo.core.metadata.schema.TabletDeletedException;
import org.apache.accumulo.core.metadata.schema.TabletMetadata;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.LocationType;
@@ -664,37 +664,22 @@ public class TableOperationsImpl extends
TableOperationsHelper {
private List _listSplits(String tableName)
throws TableNotFoundException, AccumuloSecurityException {
+
TableId tableId = context.getTableId(tableName);
-TreeMap tabletLocations = new TreeMap<>();
+
while (true) {
try {
-tabletLocations.clear();
-// the following method throws AccumuloException for some conditions
that should be retried
-MetadataServicer.forTableId(context,
tableId).getTabletLocations(tabletLocations);
-break;
- } catch (AccumuloSecurityException ase) {
-throw ase;
- } catch (Exception e) {
+return
context.getAmple().readTablets().forTable(tableId).fetch(PREV_ROW).checkConsistency()
+.build().stream().map(tm ->
tm.getExtent().endRow()).filter(Objects::nonNull)
+.collect(Collectors.toList());
+ } catch (TabletDeletedException tde) {
+// see if the table was deleted
context.requireTableExists(tableId, tableName);
-
-if (e instanceof RuntimeException && e.getCause() instanceof
AccumuloSecurityException) {
- throw (AccumuloSecurityException) e.getCause();
-}
-
-log.info("{} ... retrying ...", e, e);
+log.debug("A merge happened while trying to list splits for {} {},
retrying ", tableName,
+tableId, tde);
sleepUninterruptibly(3, SECONDS);
}
}
-
-ArrayList endRows = new ArrayList<>(tabletLocations.size());
-for (KeyExtent ke : tabletLocations.keySet()) {
- if (ke.endRow() != null) {
-endRows.add(ke.endRow());
- }
-}
-
-return endRows;
-
}
/**
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
index f6042f184b..876e6ec584 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
@@ -399,6 +399,8 @@ public class TabletsMetadata implements
Iterable, AutoCloseable
/**
* Checks that the metadata table forms a linked list and automatically
backs up until it does.
+ * May cause {@link TabletDeletedException} to be thrown while reading
tablets metadata in the
+ * case where a table is deleted or merge runs concurrently with scan.
*/
Options checkConsistency();
[accumulo] branch main updated: removes vestigial cache clear in bulk import (#3339)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 2d7a63bf07 removes vestigial cache clear in bulk import (#3339)
2d7a63bf07 is described below
commit 2d7a63bf07ab99ce00609e18203d78ddeb9aa092
Author: Keith Turner
AuthorDate: Tue Apr 25 14:41:40 2023 -0400
removes vestigial cache clear in bulk import (#3339)
While working on #3337 code that cleared the table id cache for no
apparent reason was found. Investigation indicates that the cache clear
used to happen before a cache read. The cache read was removed, but the
cache clear was left behind. This commit removes the cache clear. The
following commit shows the old clear and then read.
https://github.com/apache/accumulo/blob/ff8c875b83880c1fde3562f1280d6e20bc1ac682/server/master/src/main/java/org/apache/accumulo/master/tableOps/bulkVer2/PrepBulkImport.java#L89-L95
---
.../org/apache/accumulo/manager/tableOps/bulkVer2/PrepBulkImport.java| 1 -
1 file changed, 1 deletion(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/PrepBulkImport.java
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/PrepBulkImport.java
index 62e4315d87..4fa7d09b12 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/PrepBulkImport.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/PrepBulkImport.java
@@ -95,7 +95,6 @@ public class PrepBulkImport extends ManagerRepo {
if (manager.onlineTabletServers().isEmpty()) {
return 500;
}
-manager.getContext().clearTableListCache();
return Utils.reserveHdfsDirectory(manager, bulkInfo.sourceDir, tid);
}
[accumulo] branch elasticity updated: improve code and docs related to operation id column (#3364)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 3e2ed1be89 improve code and docs related to operation id column (#3364)
3e2ed1be89 is described below
commit 3e2ed1be897936c192214c2f08f7b2cfc6c1eeea
Author: Keith Turner
AuthorDate: Mon May 1 10:54:32 2023 -0400
improve code and docs related to operation id column (#3364)
---
.../accumulo/core/metadata/schema/Ample.java | 10 ++---
.../core/metadata/schema/MetadataSchema.java | 9 +
.../core/metadata/schema/TabletMetadata.java | 15 +++-
.../metadata/{ => schema}/TabletOperationId.java | 37 +-
...bletOperation.java => TabletOperationType.java} | 2 +-
.../server/constraints/MetadataConstraints.java| 14 ++-
.../metadata/ConditionalTabletMutatorImpl.java | 8 ++--
.../server/metadata/TabletMutatorBase.java | 7 ++--
.../constraints/MetadataConstraintsTest.java | 19 ++
.../test/functional/AmpleConditionalWriterIT.java | 44 ++
10 files changed, 112 insertions(+), 53 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
index 438e78b9e1..d7a7177da2 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
@@ -37,7 +37,6 @@ import
org.apache.accumulo.core.metadata.ScanServerRefTabletFile;
import org.apache.accumulo.core.metadata.StoredTabletFile;
import org.apache.accumulo.core.metadata.TServerInstance;
import org.apache.accumulo.core.metadata.TabletFile;
-import org.apache.accumulo.core.metadata.TabletOperationId;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType;
import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location;
import org.apache.accumulo.core.tabletserver.log.LogEntry;
@@ -321,7 +320,7 @@ public interface Ample {
T deleteHostingRequested();
-T putOperation(TabletOperation operation, TabletOperationId opId);
+T putOperation(TabletOperationId opId);
T deleteOperation();
@@ -347,7 +346,9 @@ public interface Ample {
* A tablet operation is a mutually exclusive action that is running against
a tablet. Its very
* important that every conditional mutation specifies requirements about
operations in order to
* satisfy the mutual exclusion goal. This interface forces those
requirements to specified by
- * making it the only choice avialable before specifying other tablet
requirements or mutations.
+ * making it the only choice available before specifying other tablet
requirements or mutations.
+ *
+ * @see MetadataSchema.TabletsSection.ServerColumnFamily#OPID_COLUMN
*/
interface OperationRequirements {
@@ -355,8 +356,7 @@ public interface Ample {
* Require a specific operation with a unique id is present. This would be
normally be called by
* the code executing that operation.
*/
-ConditionalTabletMutator requireOperation(TabletOperation operation,
-TabletOperationId operationId);
+ConditionalTabletMutator requireOperation(TabletOperationId operationId);
/**
* Require that no mutually exclusive operations are runnnig against this
tablet.
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
index f387c6b20c..793d36d41d 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/MetadataSchema.java
@@ -249,6 +249,15 @@ public class MetadataSchema {
public static final String LOCK_QUAL = "lock";
public static final ColumnFQ LOCK_COLUMN = new ColumnFQ(NAME, new
Text(LOCK_QUAL));
+ /**
+ * This column is used to indicate an operation is running that needs
exclusive access to read
+ * and write to a tablet. The value uniquely identifies a FATE operation
that is running and
+ * needs the exclusive access. All tablet updates must either ensure
this column is absent or
+ * in the case of a FATE operation that set it ensure the value contains
their FATE
+ * transaction id. When a FATE operation wants to set this column it
must ensure its absent
+ * before setting it. Once a FATE operation has successfully set the
column then no other
+ * tablet update should succeed.
+ */
public static final String OPID_QUAL = "opid";
public static final ColumnFQ OPID_COLUMN = new ColumnFQ(NAME, new
Text(OPID_QUAL));
}
diff --git
a/core/src/main/jav
[accumulo] 01/01: Merge remote-tracking branch 'upstream/1.10' into 2.1
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 81a2581ead2323326d2c89472922ff078ec42382
Merge: 0f060b9cb0 d925db128b
Author: Keith Turner
AuthorDate: Thu Mar 30 20:08:18 2023 -0400
Merge remote-tracking branch 'upstream/1.10' into 2.1
.../java/org/apache/accumulo/tserver/tablet/Tablet.java | 16 ++--
1 file changed, 10 insertions(+), 6 deletions(-)
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 5c4b7d40db,9cd5cbd968..fbdec25e97
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@@ -43,11 -45,9 +44,10 @@@ import java.util.concurrent.ConcurrentS
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
--import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
import org.apache.accumulo.core.Constants;
import org.apache.accumulo.core.client.Durability;
@@@ -1400,10 -1875,12 +1404,12 @@@ public class Tablet extends TabletBase
}
newComputation = new SplitComputations(files, midpoint, lastRow);
+
+ lastSplitComputation = new SoftReference<>(newComputation);
} catch (IOException e) {
- lastSplitComputation.set(null);
+ lastSplitComputation.clear();
log.error("Failed to compute split information from files " +
e.getMessage());
-return Optional.absent();
+return Optional.empty();
} finally {
splitComputationLock.unlock();
}
[accumulo] branch 2.1 updated (0f060b9cb0 -> 81a2581ead)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git from 0f060b9cb0 Convert Bulk Import metadata actions to use Ample (#3255) add d925db128b fixes #3260 by making cached split info a softref (#3261) new 81a2581ead Merge remote-tracking branch 'upstream/1.10' into 2.1 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../java/org/apache/accumulo/tserver/tablet/Tablet.java | 16 ++-- 1 file changed, 10 insertions(+), 6 deletions(-)
[accumulo] branch 1.10 updated: fixes #3260 by making cached split info a softref (#3261)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.10 by this push:
new d925db128b fixes #3260 by making cached split info a softref (#3261)
d925db128b is described below
commit d925db128b568d61fd67647e5e033dfccf26f8dc
Author: Keith Turner
AuthorDate: Thu Mar 30 19:54:50 2023 -0400
fixes #3260 by making cached split info a softref (#3261)
The changes in #3249 introduce per tablet caching of split information so
that it does not need to be recomputed. If a tablet happens to have large keys
and those end up getting cached by the new code in #3249 then it could cause
memory pressure or OOMEs in the tablet server. This commit attempts to avoid
those problems by using a softref to store the cached information.
---
.../java/org/apache/accumulo/tserver/tablet/Tablet.java | 15 ++-
1 file changed, 10 insertions(+), 5 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 580616ee3f..9cd5cbd968 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -24,6 +24,7 @@ import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.lang.ref.SoftReference;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -1821,8 +1822,12 @@ public class Tablet implements TabletCommitter {
}
}
- private AtomicReference lastSplitComputation = new
AtomicReference<>();
- private Lock splitComputationLock = new ReentrantLock();
+ // The following caches keys from users files needed to compute a tablets
split point. This cached
+ // data could potentially be large and is therefore stored using a soft
refence so the Java GC can
+ // release it if needed. If the cached information is not there it can
always be recomputed.
+ private volatile SoftReference lastSplitComputation =
+ new SoftReference<>(null);
+ private final Lock splitComputationLock = new ReentrantLock();
/**
* Computes split point information from files when a tablets set of files
changes. Do not call
@@ -1870,16 +1875,16 @@ public class Tablet implements TabletCommitter {
}
newComputation = new SplitComputations(files, midpoint, lastRow);
+
+lastSplitComputation = new SoftReference<>(newComputation);
} catch (IOException e) {
-lastSplitComputation.set(null);
+lastSplitComputation.clear();
log.error("Failed to compute split information from files " +
e.getMessage());
return Optional.absent();
} finally {
splitComputationLock.unlock();
}
- lastSplitComputation.set(newComputation);
-
return Optional.of(newComputation);
} else {
// some other thread seems to be working on split, let the other thread
work on it
[accumulo] branch main updated (639e83549f -> b3d7a61317)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 639e83549f Merge branch '2.1' add 0f060b9cb0 Convert Bulk Import metadata actions to use Ample (#3255) add d925db128b fixes #3260 by making cached split info a softref (#3261) add 81a2581ead Merge remote-tracking branch 'upstream/1.10' into 2.1 new b3d7a61317 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/metadata/schema/Ample.java | 30 .../accumulo/server/metadata/ServerAmpleImpl.java | 89 +- .../accumulo/server/util/MetadataTableUtil.java| 48 .../manager/tableOps/bulkVer1/BulkImport.java | 3 +- .../tableOps/bulkVer1/CleanUpBulkImport.java | 11 ++- .../manager/tableOps/bulkVer2/BulkImportMove.java | 4 +- .../tableOps/bulkVer2/CleanUpBulkImport.java | 11 ++- .../org/apache/accumulo/tserver/tablet/Tablet.java | 16 ++-- 8 files changed, 120 insertions(+), 92 deletions(-)
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit b3d7a613177be5a6d8a94fe2145c6d08106599cc
Merge: 639e83549f 81a2581ead
Author: Keith Turner
AuthorDate: Thu Mar 30 20:12:59 2023 -0400
Merge branch '2.1'
.../accumulo/core/metadata/schema/Ample.java | 30
.../accumulo/server/metadata/ServerAmpleImpl.java | 89 +-
.../accumulo/server/util/MetadataTableUtil.java| 48
.../manager/tableOps/bulkVer1/BulkImport.java | 3 +-
.../tableOps/bulkVer1/CleanUpBulkImport.java | 11 ++-
.../manager/tableOps/bulkVer2/BulkImportMove.java | 4 +-
.../tableOps/bulkVer2/CleanUpBulkImport.java | 11 ++-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 16 ++--
8 files changed, 120 insertions(+), 92 deletions(-)
diff --cc
server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
index 33c0a415a5,08871bd97b..15133cf345
---
a/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/util/MetadataTableUtil.java
@@@ -62,9 -61,8 +61,8 @@@ import org.apache.accumulo.core.data.Ra
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
- import org.apache.accumulo.core.fate.FateTxId;
-import org.apache.accumulo.core.fate.zookeeper.ServiceLock;
import org.apache.accumulo.core.gc.ReferenceFile;
+import org.apache.accumulo.core.lock.ServiceLock;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.StoredTabletFile;
diff --cc
server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/BulkImportMove.java
index d6822edf56,5ace9ccb6f..8f1e75782f
---
a/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/BulkImportMove.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/tableOps/bulkVer2/BulkImportMove.java
@@@ -103,11 -102,12 +102,10 @@@ class BulkImportMove extends ManagerRep
*/
private void moveFiles(long tid, Path sourceDir, Path bulkDir, Manager
manager,
final VolumeManager fs, Map renames) throws Exception {
- MetadataTableUtil.addBulkLoadInProgressFlag(manager.getContext(),
+ manager.getContext().getAmple().addBulkLoadInProgressFlag(
"/" + bulkDir.getParent().getName() + "/" + bulkDir.getName(), tid);
-
AccumuloConfiguration aConf = manager.getConfiguration();
-@SuppressWarnings("deprecation")
-int workerCount = aConf.getCount(
-aConf.resolve(Property.MANAGER_RENAME_THREADS,
Property.MANAGER_BULK_RENAME_THREADS));
+int workerCount = aConf.getCount(Property.MANAGER_RENAME_THREADS);
Map oldToNewMap = new HashMap<>();
String fmtTid = FateTxId.formatTid(tid);
[accumulo] branch 2.1 updated: Convert Bulk Import metadata actions to use Ample (#3255)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 0f060b9cb0 Convert Bulk Import metadata actions to use Ample (#3255)
0f060b9cb0 is described below
commit 0f060b9cb0e0811398a88b17ff62fb070ef6a463
Author: Daniel Roberts
AuthorDate: Thu Mar 30 18:27:07 2023 -0400
Convert Bulk Import metadata actions to use Ample (#3255)
Refactors the addBulkImportInProgressFlag, removeBulkImportInProgressFlag,
and removeBulkLoadEntries methods and moves them to ServerAmpleImpl
from the MetadataTableUtil class.
Removes the private createWriter method in favor of just calling
context directly with the datalevel.
---
.../accumulo/core/metadata/schema/Ample.java | 30
.../accumulo/server/metadata/ServerAmpleImpl.java | 89 +-
.../accumulo/server/util/MetadataTableUtil.java| 48
.../manager/tableOps/bulkVer1/BulkImport.java | 3 +-
.../tableOps/bulkVer1/CleanUpBulkImport.java | 11 ++-
.../manager/tableOps/bulkVer2/BulkImportMove.java | 4 +-
.../tableOps/bulkVer2/CleanUpBulkImport.java | 11 ++-
7 files changed, 110 insertions(+), 86 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
index 7e1c5163a4..7db46c77a0 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
@@ -339,4 +339,34 @@ public interface Ample {
default void deleteScanServerFileReferences(String serverAddress, UUID
serverSessionId) {
throw new UnsupportedOperationException();
}
+
+ /**
+ * Create a Bulk Load In Progress flag in the metadata table
+ *
+ * @param path The bulk directory filepath
+ * @param fateTxid The id of the Bulk Import Fate operation.
+ */
+ default void addBulkLoadInProgressFlag(String path, long fateTxid) {
+throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Remove a Bulk Load In Progress flag from the metadata table.
+ *
+ * @param path The bulk directory filepath
+ */
+ default void removeBulkLoadInProgressFlag(String path) {
+throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Remove all the Bulk Load transaction ids from a given table's metadata
+ *
+ * @param tableId Table ID for transaction removals
+ * @param tid Transaction ID to remove
+ */
+ default void removeBulkLoadEntries(TableId tableId, long tid) {
+throw new UnsupportedOperationException();
+ }
+
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
index 3f23fca6ab..d23917f0a3 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/metadata/ServerAmpleImpl.java
@@ -24,6 +24,7 @@ import static
org.apache.accumulo.server.util.MetadataTableUtil.EMPTY_TEXT;
import java.util.Collection;
import java.util.Iterator;
+import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
@@ -33,13 +34,17 @@ import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.IsolatedScanner;
import org.apache.accumulo.core.client.MutationsRejectedException;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Mutation;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.data.Value;
import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.fate.FateTxId;
import org.apache.accumulo.core.gc.ReferenceFile;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
@@ -50,10 +55,12 @@ import org.apache.accumulo.core.metadata.schema.Ample;
import org.apache.accumulo.core.metadata.schema.AmpleImpl;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionFinalState;
import org.apache.accumulo.core.metadata.schema.ExternalCompactionId;
+import org.apache.accumulo.core.metadata.schema.MetadataSchema.BlipSection;
import org.apache.accumulo.core.metadata.schema.MetadataSchema.DeletesSection;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.DeletesSection.SkewedKeyValue;
import
org.apache.accumulo.core.metadata.schema.MetadataSchema.ExternalCompactionSection;
import
[accumulo-website] branch asf-site updated (4df3a3e9 -> 1d7f9ac7)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/accumulo-website.git from 4df3a3e9 Automatic Site Publish by Buildbot add 25dd01c0 Automatic Site Publish by Buildbot add 1d7f9ac7 Automatic Site Publish by Buildbot No new revisions were added by this update. Summary of changes: output/feed.xml | 4 ++-- output/people/index.html | 2 +- output/release/accumulo-2.1.1/index.html | 1 + output/search_data.json | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-)
[accumulo] branch main updated: Performs some minor cleanup of TabletLocator (#3271)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 1409bc0fdd Performs some minor cleanup of TabletLocator (#3271)
1409bc0fdd is described below
commit 1409bc0fdd8fa9b262d8de5a388af99ef5deb61e
Author: Keith Turner
AuthorDate: Mon Apr 3 09:44:55 2023 -0400
Performs some minor cleanup of TabletLocator (#3271)
This commit has no changes in functionality, it only contains some
minor code cleanup of the TabletLocator code.
The following changes were made.
* Encapsulated fields in TabletLocation class
* Replaced a Callable interface with a Supplier interface to avoid the
uneeded exception thrown by Callable
* Removed an uneeded compareTo method from TabletLocation class
---
.../core/clientImpl/RootTabletLocator.java | 6 +-
.../core/clientImpl/SyncingTabletLocator.java | 20 ++-
.../core/clientImpl/TableOperationsImpl.java | 14 ++---
.../accumulo/core/clientImpl/TabletLocator.java| 44 +++
.../core/clientImpl/TabletLocatorImpl.java | 65 +++---
.../accumulo/core/clientImpl/ThriftScanner.java| 59 ++--
.../apache/accumulo/core/clientImpl/Writer.java| 10 ++--
.../core/metadata/MetadataLocationObtainer.java| 20 +++
.../core/clientImpl/TabletLocatorImplTest.java | 18 +++---
.../accumulo/server/client/BulkImporter.java | 14 ++---
.../accumulo/server/client/BulkImporterTest.java | 19 ---
.../accumulo/test/functional/BulkFailureIT.java| 2 +-
.../accumulo/test/manager/SuspendedTabletsIT.java | 4 +-
13 files changed, 144 insertions(+), 151 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/RootTabletLocator.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/RootTabletLocator.java
index a7609c9ba3..a24a236998 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/RootTabletLocator.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/RootTabletLocator.java
@@ -56,11 +56,11 @@ public class RootTabletLocator extends TabletLocator {
Map> binnedMutations, List failures) {
TabletLocation rootTabletLocation = getRootTabletLocation(context);
if (rootTabletLocation != null) {
- TabletServerMutations tsm = new
TabletServerMutations<>(rootTabletLocation.tablet_session);
+ var tsm = new
TabletServerMutations(rootTabletLocation.getTserverSession());
for (T mutation : mutations) {
tsm.addMutation(RootTable.EXTENT, mutation);
}
- binnedMutations.put(rootTabletLocation.tablet_location, tsm);
+ binnedMutations.put(rootTabletLocation.getTserverLocation(), tsm);
} else {
failures.addAll(mutations);
}
@@ -73,7 +73,7 @@ public class RootTabletLocator extends TabletLocator {
TabletLocation rootTabletLocation = getRootTabletLocation(context);
if (rootTabletLocation != null) {
for (Range range : ranges) {
-TabletLocatorImpl.addRange(binnedRanges,
rootTabletLocation.tablet_location,
+TabletLocatorImpl.addRange(binnedRanges,
rootTabletLocation.getTserverLocation(),
RootTable.EXTENT, range);
}
return Collections.emptyList();
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/SyncingTabletLocator.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/SyncingTabletLocator.java
index 8f6daed955..ff9a75cf3f 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/SyncingTabletLocator.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/SyncingTabletLocator.java
@@ -21,7 +21,7 @@ package org.apache.accumulo.core.clientImpl;
import java.util.Collection;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.Callable;
+import java.util.function.Supplier;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
@@ -43,16 +43,11 @@ public class SyncingTabletLocator extends TabletLocator {
private static final Logger log =
LoggerFactory.getLogger(SyncingTabletLocator.class);
private volatile TabletLocator locator;
- private final Callable getLocatorFunction;
+ private final Supplier getLocatorFunction;
- public SyncingTabletLocator(Callable getLocatorFunction) {
+ public SyncingTabletLocator(Supplier getLocatorFunction) {
this.getLocatorFunction = getLocatorFunction;
-try {
- this.locator = getLocatorFunction.call();
-} catch (Exception e) {
- log.error("Problem obtaining TabletLocator", e);
- throw new RuntimeException(e);
-}
+this.locator = getLocatorFunction.get();
}
public SyncingTabletLocator(final ClientContext context, final TableId
tableId) {
@@ -64
[accumulo-website] branch main updated: Fixes #379 creates emeritus PMC table on people page (#380)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo-website.git
The following commit(s) were added to refs/heads/main by this push:
new 04f183ab Fixes #379 creates emeritus PMC table on people page (#380)
04f183ab is described below
commit 04f183aba9d18f11fd8e852fd0348b47a62fb1e7
Author: Keith Turner
AuthorDate: Fri Mar 31 15:49:47 2023 -0400
Fixes #379 creates emeritus PMC table on people page (#380)
---
pages/people.md | 10 --
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/pages/people.md b/pages/people.md
index e7b2476a..263bff0d 100644
--- a/pages/people.md
+++ b/pages/people.md
@@ -25,7 +25,6 @@ $(function() {
| adamjshook| Adam J. Shook |
[Datacatessen][DATACATESS] | [ET][ET] |
| afuchs| Adam Fuchs|
[sqrrl][SQRRL] | [ET][ET] |
| alerman | Adam Lerman | [Applied
Technology Group][ATG]| [ET][ET] |
-| arvindsh | Arvind Shyamsundar|
[Microsoft][MICROSOFT] | [PT][PT] |
| bhavanki | Bill Havanki |
[Cloudera][CLOUDERA] | [ET][ET] |
| billie| Billie Rinaldi|
[Microsoft][MICROSOFT] | [ET][ET] |
| bimargulies | Benson Margulies | [Basis
Technology Corp.][BASISTECH]| [ET][ET] |
@@ -49,7 +48,6 @@ $(function() {
| kturner | [Keith Turner](https://github.com/keith-turner) |
[Wrench.io, LLC][WRENCH] | [ET][ET] |
| lstavarez | [Luis Tavarez](https://github.com/lstav) |
| [ET][ET] |
| mdrob | Mike Drob |
[Cloudera][CLOUDERA] | [ET][ET] |
-| medined | David Medinets|
| |
| mjwall| Michael Wall | [Arctic
Slope Regional Corp.][ASRC]| [ET][ET] |
| mmiller | [Michael Miller](https://github.com/milleruntime) |
[Centroid, LLC][CENTROID] | [ET][ET] |
| mwalch| [Mike Walch](https://github.com/mikewalch)|
[Peterson Technologies][PETERSON] | [ET][ET] |
@@ -62,6 +60,14 @@ $(function() {
| vikrams | Vikram Srivastava |
[Cloudera][CLOUDERA] | [PT][PT] |
| vines | John Vines|
[sqrrl][SQRRL] | [ET][ET] |
+## Committers Only (PMC Emeritus)
+
+{: .table .table-striped #pmc}
+| apache id | name |
organization | timezone |
+|---|---||--|
+| arvindsh | Arvind Shyamsundar|
[Microsoft][MICROSOFT] | [PT][PT] |
+| medined | David Medinets|
| |
+
## Contributors
GitHub also has a [contributor list][github-contributors] based on commits.
[accumulo] branch main updated: centralizes wrapping per file iterators based on metadata (#3259)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 8c65e026be centralizes wrapping per file iterators based on metadata
(#3259)
8c65e026be is described below
commit 8c65e026be2db633d8f2a622aabf05db1c2d6c15
Author: Keith Turner
AuthorDate: Fri Mar 31 14:56:18 2023 -0400
centralizes wrapping per file iterators based on metadata (#3259)
---
.../core/metadata/schema/DataFileValue.java | 21 +
.../accumulo/server/compaction/FileCompactor.java | 10 --
.../org/apache/accumulo/server/fs/FileManager.java | 10 --
3 files changed, 29 insertions(+), 12 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java
index 67c5e1518a..fe770d96c0 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/DataFileValue.java
@@ -21,6 +21,8 @@ package org.apache.accumulo.core.metadata.schema;
import static java.nio.charset.StandardCharsets.UTF_8;
import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iteratorsImpl.system.InterruptibleIterator;
+import org.apache.accumulo.core.iteratorsImpl.system.TimeSettingIterator;
public class DataFileValue {
private long size;
@@ -114,4 +116,23 @@ public class DataFileValue {
}
this.time = time;
}
+
+ /**
+ * @return true if {@link #wrapFileIterator} would wrap a given iterator,
false otherwise.
+ */
+ public boolean willWrapIterator() {
+return isTimeSet();
+ }
+
+ /**
+ * Use per file information from the metadata table to wrap the raw iterator
over a file with
+ * iterators that may take action based on data set in the metadata table.
+ */
+ public InterruptibleIterator wrapFileIterator(InterruptibleIterator iter) {
+if (isTimeSet()) {
+ return new TimeSettingIterator(iter, getTime());
+} else {
+ return iter;
+}
+ }
}
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
b/server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
index f8ba0011c9..3b6b84a7d0 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/compaction/FileCompactor.java
@@ -52,8 +52,8 @@ import
org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iteratorsImpl.IteratorConfigUtil;
import
org.apache.accumulo.core.iteratorsImpl.system.ColumnFamilySkippingIterator;
import org.apache.accumulo.core.iteratorsImpl.system.DeletingIterator;
+import org.apache.accumulo.core.iteratorsImpl.system.InterruptibleIterator;
import org.apache.accumulo.core.iteratorsImpl.system.MultiIterator;
-import org.apache.accumulo.core.iteratorsImpl.system.TimeSettingIterator;
import org.apache.accumulo.core.metadata.MetadataTable;
import org.apache.accumulo.core.metadata.RootTable;
import org.apache.accumulo.core.metadata.StoredTabletFile;
@@ -344,12 +344,10 @@ public class FileCompactor implements
Callable {
readers.add(reader);
-SortedKeyValueIterator iter = new
ProblemReportingIterator(context,
-extent.tableId(), mapFile.getPathStr(), false, reader);
+InterruptibleIterator iter = new ProblemReportingIterator(context,
extent.tableId(),
+mapFile.getPathStr(), false, reader);
-if (filesToCompact.get(mapFile).isTimeSet()) {
- iter = new TimeSettingIterator(iter,
filesToCompact.get(mapFile).getTime());
-}
+iter = filesToCompact.get(mapFile).wrapFileIterator(iter);
iters.add(iter);
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/fs/FileManager.java
b/server/base/src/main/java/org/apache/accumulo/server/fs/FileManager.java
index 2284c81666..b5dce00ec0 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/fs/FileManager.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/fs/FileManager.java
@@ -46,7 +46,6 @@ import
org.apache.accumulo.core.iterators.SortedKeyValueIterator;
import org.apache.accumulo.core.iteratorsImpl.system.InterruptibleIterator;
import org.apache.accumulo.core.iteratorsImpl.system.SourceSwitchingIterator;
import
org.apache.accumulo.core.iteratorsImpl.system.SourceSwitchingIterator.DataSource;
-import org.apache.accumulo.core.iteratorsImpl.system.TimeSettingIterator;
import org.apache.accumulo.core.metadata.TabletFile;
import org.apache.accumulo.core.metadata.schema.DataFileValue;
import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
@@ -509,7 +508,8 @@ public class FileManager
[accumulo] 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 293feb220ec071dcb81cd50c69bcdc20a4a45ff6 Merge: 6a56f24146 8c65e026be Author: Keith Turner AuthorDate: Fri Mar 31 17:27:37 2023 -0400 Merge branch 'main' into elasticity .../accumulo/core/metadata/schema/Ample.java | 30 .../core/metadata/schema/DataFileValue.java| 21 + pom.xml| 15 ++-- .../accumulo/server/compaction/FileCompactor.java | 10 +-- .../org/apache/accumulo/server/fs/FileManager.java | 10 +-- .../accumulo/server/metadata/ServerAmpleImpl.java | 89 +- .../accumulo/server/util/MetadataTableUtil.java| 48 .../manager/tableOps/bulkVer1/BulkImport.java | 3 +- .../tableOps/bulkVer1/CleanUpBulkImport.java | 11 ++- .../manager/tableOps/bulkVer2/BulkImportMove.java | 4 +- .../tableOps/bulkVer2/CleanUpBulkImport.java | 11 ++- .../org/apache/accumulo/tserver/tablet/Tablet.java | 16 ++-- 12 files changed, 159 insertions(+), 109 deletions(-)
[accumulo] branch elasticity updated (6a56f24146 -> 293feb220e)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 6a56f24146 Merge pull request #3250 from dlmarion/elasticity-on-demand-tables add be582a609e Updated Hadoop version from 3.3.4 to 3.3.5 (#3258) add ca4d43f6df Merge branch '2.1' add 0f27c32ac9 Set root location property for paths to contrib files (#3266) add 639e83549f Merge branch '2.1' add 0f060b9cb0 Convert Bulk Import metadata actions to use Ample (#3255) add d925db128b fixes #3260 by making cached split info a softref (#3261) add 81a2581ead Merge remote-tracking branch 'upstream/1.10' into 2.1 add b3d7a61317 Merge branch '2.1' add 8c65e026be centralizes wrapping per file iterators based on metadata (#3259) new 293feb220e Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/metadata/schema/Ample.java | 30 .../core/metadata/schema/DataFileValue.java| 21 + pom.xml| 15 ++-- .../accumulo/server/compaction/FileCompactor.java | 10 +-- .../org/apache/accumulo/server/fs/FileManager.java | 10 +-- .../accumulo/server/metadata/ServerAmpleImpl.java | 89 +- .../accumulo/server/util/MetadataTableUtil.java| 48 .../manager/tableOps/bulkVer1/BulkImport.java | 3 +- .../tableOps/bulkVer1/CleanUpBulkImport.java | 11 ++- .../manager/tableOps/bulkVer2/BulkImportMove.java | 4 +- .../tableOps/bulkVer2/CleanUpBulkImport.java | 11 ++- .../org/apache/accumulo/tserver/tablet/Tablet.java | 16 ++-- 12 files changed, 159 insertions(+), 109 deletions(-)
[accumulo-website] branch asf-site updated (1d7f9ac7 -> 9d15a36f)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch asf-site in repository https://gitbox.apache.org/repos/asf/accumulo-website.git from 1d7f9ac7 Automatic Site Publish by Buildbot add 9d15a36f Automatic Site Publish by Buildbot No new revisions were added by this update. Summary of changes: output/feed.xml | 4 ++-- output/people/index.html | 39 +++ 2 files changed, 29 insertions(+), 14 deletions(-)
[accumulo] branch 2.1 updated: Fixes a minor scan server bug and adds some since tags (#3277)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new fbe647e925 Fixes a minor scan server bug and adds some since tags
(#3277)
fbe647e925 is described below
commit fbe647e925a26f978a75d38e7e5af8948c33f739
Author: Keith Turner
AuthorDate: Thu Apr 6 22:08:37 2023 -0400
Fixes a minor scan server bug and adds some since tags (#3277)
The pluggable scan server selector provided by Accumulo had a minor
bug in its configration parsing. It expected the default profile to
always come first in the config, even though it could occur any
position.
Also some since tags were missing the for scan servers SPIs, added
those.
---
.../accumulo/core/spi/scan/ConfigurableScanServerSelector.java| 8 +---
.../org/apache/accumulo/core/spi/scan/ScanServerSelections.java | 6 ++
.../core/spi/scan/ConfigurableScanServerSelectorTest.java | 4 +++-
3 files changed, 14 insertions(+), 4 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/scan/ConfigurableScanServerSelector.java
b/core/src/main/java/org/apache/accumulo/core/spi/scan/ConfigurableScanServerSelector.java
index 42f031a370..2e792180cc 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/scan/ConfigurableScanServerSelector.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/scan/ConfigurableScanServerSelector.java
@@ -144,6 +144,8 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
* will keep increasing the busy timeout by multiplying 8 until the maximum of
20 minutes is
* reached. For this profile it will choose from scan servers in the group
{@literal lowcost}.
*
+ *
+ * @since 2.1.0
*/
public class ConfigurableScanServerSelector implements ScanServerSelector {
@@ -284,10 +286,10 @@ public class ConfigurableScanServerSelector implements
ScanServerSelector {
defaultProfile = prof;
}
+}
- if (defaultProfile == null) {
-throw new IllegalArgumentException("No default profile specified");
- }
+if (defaultProfile == null) {
+ throw new IllegalArgumentException("No default profile specified");
}
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/spi/scan/ScanServerSelections.java
b/core/src/main/java/org/apache/accumulo/core/spi/scan/ScanServerSelections.java
index c5dbab2f2c..0236e21c82 100644
---
a/core/src/main/java/org/apache/accumulo/core/spi/scan/ScanServerSelections.java
+++
b/core/src/main/java/org/apache/accumulo/core/spi/scan/ScanServerSelections.java
@@ -22,6 +22,12 @@ import java.time.Duration;
import org.apache.accumulo.core.data.TabletId;
+/**
+ * Returned by {@link
ScanServerSelector#selectServers(ScanServerSelector.SelectorParameters)} to
+ * specify what scan servers to use and how to use them.
+ *
+ * @since 2.1.0
+ */
public interface ScanServerSelections {
/**
diff --git
a/core/src/test/java/org/apache/accumulo/core/spi/scan/ConfigurableScanServerSelectorTest.java
b/core/src/test/java/org/apache/accumulo/core/spi/scan/ConfigurableScanServerSelectorTest.java
index d4448fb08d..376cfba81d 100644
---
a/core/src/test/java/org/apache/accumulo/core/spi/scan/ConfigurableScanServerSelectorTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/spi/scan/ConfigurableScanServerSelectorTest.java
@@ -294,8 +294,10 @@ public class ConfigurableScanServerSelectorTest {
"{'scanTypeActivations':['mega'],'maxBusyTimeout':'60m','busyTimeoutMultiplier':2,
"
+ "'attemptPlans':[{'servers':'100%', 'busyTimeout':'10m'}]}";
+// Intentionally put the default profile in 2nd position. There was a bug
where config parsing
+// would fail if the default did not come first.
var opts = Map.of("profiles",
-"[" + defaultProfile + ", " + profile1 + "," + profile2 +
"]".replace('\'', '"'));
+"[" + profile1 + ", " + defaultProfile + "," + profile2 +
"]".replace('\'', '"'));
runBusyTest(1000, 0, 5, 5, opts);
runBusyTest(1000, 1, 20, 33, opts);
[accumulo] 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 7f49667f1347ba446d0f8e31271df8ba7ee48397 Merge: a10934437e fbe647e925 Author: Keith Turner AuthorDate: Thu Apr 6 22:09:35 2023 -0400 Merge remote-tracking branch 'upstream/2.1' .../accumulo/core/spi/scan/ConfigurableScanServerSelector.java| 8 +--- .../org/apache/accumulo/core/spi/scan/ScanServerSelections.java | 6 ++ .../core/spi/scan/ConfigurableScanServerSelectorTest.java | 4 +++- 3 files changed, 14 insertions(+), 4 deletions(-)
[accumulo] branch main updated (a10934437e -> 7f49667f13)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from a10934437e Merge branch '2.1' add fbe647e925 Fixes a minor scan server bug and adds some since tags (#3277) new 7f49667f13 Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/core/spi/scan/ConfigurableScanServerSelector.java| 8 +--- .../org/apache/accumulo/core/spi/scan/ScanServerSelections.java | 6 ++ .../core/spi/scan/ConfigurableScanServerSelectorTest.java | 4 +++- 3 files changed, 14 insertions(+), 4 deletions(-)
[accumulo] 01/01: Merge remote-tracking branch 'upstream/main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 41a3053d539629faf1c3342dd4a85da8cb072215
Merge: 7a54f64282 8b76290644
Author: Keith Turner
AuthorDate: Wed Apr 5 16:22:21 2023 -0400
Merge remote-tracking branch 'upstream/main' into elasticity
.../apache/accumulo/core/client/ScannerBase.java | 1 +
.../core/clientImpl/RootTabletLocator.java | 6 +-
.../core/clientImpl/SyncingTabletLocator.java | 23 +--
.../core/clientImpl/TableOperationsImpl.java | 14 +-
.../accumulo/core/clientImpl/TabletLocator.java| 44 +++---
.../core/clientImpl/TabletLocatorImpl.java | 65
.../accumulo/core/clientImpl/ThriftScanner.java| 59 +++
.../apache/accumulo/core/clientImpl/Writer.java| 10 +-
.../core/metadata/MetadataLocationObtainer.java| 20 +--
.../accumulo/core/security/Authorizations.java | 52 +-
.../core/clientImpl/TabletLocatorImplTest.java | 18 +--
.../hadoop/mapreduce/InputFormatBuilder.java | 2 +
.../accumulo/server/client/BulkImporter.java | 14 +-
.../server/conf/ServerConfigurationFactory.java| 23 ++-
.../accumulo/server/util/MetadataTableUtil.java| 4 -
.../accumulo/server/client/BulkImporterTest.java | 19 +--
.../accumulo/test/functional/BulkFailureIT.java| 2 +-
.../accumulo/test/manager/SuspendedTabletsIT.java | 4 +-
.../apache/accumulo/test/shell/ShellServerIT.java | 175 -
19 files changed, 306 insertions(+), 249 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
index 6fa7716dc6,6d0f2aa735..76afd84fc1
---
a/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/MetadataLocationObtainer.java
@@@ -87,9 -87,9 +87,9 @@@ public class MetadataLocationObtainer i
OpTimer timer = null;
if (log.isTraceEnabled()) {
-log.trace("tid={} Looking up in {} row={} extent={} tserver={}",
+log.trace("tid={} Looking up in {} row={} stopRow={} extent={}
tserver={}",
- Thread.currentThread().getId(), src.tablet_extent.tableId(),
TextUtil.truncate(row),
- TextUtil.truncate(stopRow), src.tablet_extent,
src.tablet_location);
+ Thread.currentThread().getId(), src.getExtent().tableId(),
TextUtil.truncate(row),
-src.getExtent(), src.getTserverLocation());
++TextUtil.truncate(stopRow), src.getExtent(),
src.getTserverLocation());
timer = new OpTimer().start();
}
[accumulo] branch elasticity updated (7a54f64282 -> 41a3053d53)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 7a54f64282 Fix onDemand javadoc and comments (#3268) add 1409bc0fdd Performs some minor cleanup of TabletLocator (#3271) add 7ad91eb32a adds missing since tags (#3275) add 9b7c7bd91a Merge remote-tracking branch 'upstream/2.1' add 8e22bf3e39 Optimize internal data structures in Authorizations (#3276) add aff0cf9ca4 Merge remote-tracking branch 'upstream/1.10' into 2.1 add 9303d5c489 Merge remote-tracking branch 'upstream/2.1' add 1ea29605ee Clear ServerConfigurationFactory cache on ZK notifications (#3245) add b6b4859f71 Remove unused private method add b7a30e9da6 Merge branch '2.1' add 8b76290644 Remove unused private variable new 41a3053d53 Merge remote-tracking branch 'upstream/main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../apache/accumulo/core/client/ScannerBase.java | 1 + .../core/clientImpl/RootTabletLocator.java | 6 +- .../core/clientImpl/SyncingTabletLocator.java | 23 +-- .../core/clientImpl/TableOperationsImpl.java | 14 +- .../accumulo/core/clientImpl/TabletLocator.java| 44 +++--- .../core/clientImpl/TabletLocatorImpl.java | 65 .../accumulo/core/clientImpl/ThriftScanner.java| 59 +++ .../apache/accumulo/core/clientImpl/Writer.java| 10 +- .../core/metadata/MetadataLocationObtainer.java| 20 +-- .../accumulo/core/security/Authorizations.java | 52 +- .../core/clientImpl/TabletLocatorImplTest.java | 18 +-- .../hadoop/mapreduce/InputFormatBuilder.java | 2 + .../accumulo/server/client/BulkImporter.java | 14 +- .../server/conf/ServerConfigurationFactory.java| 23 ++- .../accumulo/server/util/MetadataTableUtil.java| 4 - .../accumulo/server/client/BulkImporterTest.java | 19 +-- .../accumulo/test/functional/BulkFailureIT.java| 2 +- .../accumulo/test/manager/SuspendedTabletsIT.java | 4 +- .../apache/accumulo/test/shell/ShellServerIT.java | 175 - 19 files changed, 306 insertions(+), 249 deletions(-)
[accumulo] branch elasticity updated: Resolves server to use for scan in a single place (#3272)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new b3d32efc87 Resolves server to use for scan in a single place (#3272)
b3d32efc87 is described below
commit b3d32efc873b6511ab49e15287e99f3218c9954b
Author: Keith Turner
AuthorDate: Wed Apr 5 16:45:42 2023 -0400
Resolves server to use for scan in a single place (#3272)
The code for resolving which tserver or sserver to use for a scan was
spread out across multiple methods responsible for executing a scan.
Pulled the code to resolve which server to use into a single place in
the code that executes a scan.
Also introduced a new class to represent the server and server type
(sserver or tserver) used to process a scan.
These changes clean up two problems in the code. First the tablet
server location class was being used to represent a scan server
with a special string placed in the tserver session field. Second
the decision to use a scan server was deeper in the scan code than
error reporting code and the resulted in the need for an odd
instance variable to remember that a scan server was used for error
reporting. Removing these two problems makes the code easier to
modify and maintain.
---
.../accumulo/core/clientImpl/ThriftScanner.java| 419 -
1 file changed, 243 insertions(+), 176 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java
index 663b3748c1..acb2e9b2bd 100644
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java
+++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/ThriftScanner.java
@@ -30,6 +30,7 @@ import java.util.EnumMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
import java.util.SortedMap;
import java.util.SortedSet;
@@ -80,6 +81,7 @@ import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
import com.google.common.net.HostAndPort;
import io.opentelemetry.api.trace.Span;
@@ -171,6 +173,44 @@ public class ThriftScanner {
throw new AccumuloException("getBatchFromServer: failed");
}
+ enum ServerType {
+TSERVER, SSERVER
+ }
+
+ static class ScanAddress {
+final String serverAddress;
+final ServerType serverType;
+final TabletLocation tabletInfo;
+
+public ScanAddress(String serverAddress, ServerType serverType,
TabletLocation tabletInfo) {
+ this.serverAddress = Objects.requireNonNull(serverAddress);
+ this.serverType = Objects.requireNonNull(serverType);
+ this.tabletInfo = Objects.requireNonNull(tabletInfo);
+}
+
+public KeyExtent getExtent() {
+ return tabletInfo.getExtent();
+}
+
+@Override
+public boolean equals(Object o) {
+ if (this == o) {
+return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+return false;
+ }
+ ScanAddress that = (ScanAddress) o;
+ return serverAddress.equals(that.serverAddress) && serverType ==
that.serverType
+ && getExtent().equals(that.getExtent());
+}
+
+@Override
+public int hashCode() {
+ return Objects.hash(serverAddress, serverType, tabletInfo);
+}
+ }
+
public static class ScanState {
boolean isolated;
@@ -189,7 +229,7 @@ public class ThriftScanner {
Authorizations authorizations;
List columns;
-TabletLocation prevLoc;
+ScanAddress prevLoc;
Long scanID;
String classLoaderContext;
@@ -207,10 +247,6 @@ public class ThriftScanner {
Duration busyTimeout;
-TabletLocation getErrorLocation() {
- return prevLoc;
-}
-
public ScanState(ClientContext context, TableId tableId, Authorizations
authorizations,
Range range, SortedSet fetchedColumns, int size,
List serverSideIteratorList,
@@ -280,10 +316,169 @@ public class ThriftScanner {
return (long) (Math.min(millis * 2, maxSleep) * (.9 + random.nextDouble()
/ 5));
}
+ private static ScanAddress getScanServerAddress(ClientContext context,
ScanState scanState,
+ TabletLocation loc) {
+Preconditions.checkArgument(scanState.runOnScanServer);
+
+ScanAddress addr = null;
+
+if (scanState.scanID != null && scanState.prevLoc != null
+&& scanState.prevLoc.serverType == ServerType.SSERVER
+&& scanState.prevLoc.getExtent().equals(loc.getExtent())) {
+ // this is the case of continuing a scan on a scan server for the same
tablet, so lets not
+ // call the scan server selector and jus
[accumulo] branch elasticity updated: Improves scan server resolution in batch scanner code. (#3273)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 149b65ce24 Improves scan server resolution in batch scanner code.
(#3273)
149b65ce24 is described below
commit 149b65ce242901ed9a579a8d2b0badc53c24ae04
Author: Keith Turner
AuthorDate: Wed Apr 5 16:46:59 2023 -0400
Improves scan server resolution in batch scanner code. (#3273)
Refactors the batch scanner code to make two improvements. First
mapping ranges to scan servers or tservers was moved under one method.
Second the process of mapping ranges to scan servers using the tablet
location cache was improved. The tablet location cache used to organized
data in a a way that was useful for immediate scans and eventual scans
had to reorganize the data. A new method was added to the tablet
location cache that passes the raw data to a consumer that can orgranize
it in any way. This new method is used for scan servers.
---
.../core/clientImpl/RootTabletLocator.java | 8 +-
.../core/clientImpl/SyncingTabletLocator.java | 8 ++
.../accumulo/core/clientImpl/TabletLocator.java| 23 +++-
.../core/clientImpl/TabletLocatorImpl.java | 32 ++---
.../TabletServerBatchReaderIterator.java | 147 ++---
.../accumulo/server/client/BulkImporterTest.java | 5 +-
6 files changed, 149 insertions(+), 74 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/RootTabletLocator.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/RootTabletLocator.java
index a24a236998..f29a27b591 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/RootTabletLocator.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/RootTabletLocator.java
@@ -27,6 +27,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.function.BiConsumer;
import org.apache.accumulo.core.Constants;
import
org.apache.accumulo.core.clientImpl.TabletLocatorImpl.TabletServerLockChecker;
@@ -67,14 +68,13 @@ public class RootTabletLocator extends TabletLocator {
}
@Override
- public List binRanges(ClientContext context, List ranges,
- Map>> binnedRanges) {
+ public List locateTablets(ClientContext context, List ranges,
+ BiConsumer rangeConsumer) {
TabletLocation rootTabletLocation = getRootTabletLocation(context);
if (rootTabletLocation != null) {
for (Range range : ranges) {
-TabletLocatorImpl.addRange(binnedRanges,
rootTabletLocation.getTserverLocation(),
-RootTable.EXTENT, range);
+rangeConsumer.accept(rootTabletLocation, range);
}
return Collections.emptyList();
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/SyncingTabletLocator.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/SyncingTabletLocator.java
index df40a21ce1..dc38d18f8a 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/SyncingTabletLocator.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/SyncingTabletLocator.java
@@ -21,6 +21,7 @@ package org.apache.accumulo.core.clientImpl;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.function.BiConsumer;
import java.util.function.Supplier;
import org.apache.accumulo.core.client.AccumuloException;
@@ -76,6 +77,13 @@ public class SyncingTabletLocator extends TabletLocator {
syncLocator().binMutations(context, mutations, binnedMutations, failures);
}
+ @Override
+ public List locateTablets(ClientContext context, List ranges,
+ BiConsumer rangeConsumer)
+ throws AccumuloException, AccumuloSecurityException,
TableNotFoundException {
+return syncLocator().locateTablets(context, ranges, rangeConsumer);
+ }
+
@Override
public List binRanges(ClientContext context, List ranges,
Map>> binnedRanges)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletLocator.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletLocator.java
index fd30d620b9..a60e63d213 100644
--- a/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletLocator.java
+++ b/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletLocator.java
@@ -26,6 +26,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.function.BiConsumer;
import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.AccumuloSecurityException;
@@ -64,10 +65,28 @@ public abstract class TabletLocator {
Map> binnedMutations, List failures)
throws AccumuloException, AccumuloSecurityException,
TableNotFoundExcepti
[accumulo] 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 9b7c7bd91a88bfed3ac5d2d2d4550e10e91dd282 Merge: 1409bc0fdd 7ad91eb32a Author: Keith Turner AuthorDate: Tue Apr 4 10:47:41 2023 -0400 Merge remote-tracking branch 'upstream/2.1' core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java | 1 + .../java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java | 2 ++ 2 files changed, 3 insertions(+)
[accumulo] branch main updated (1409bc0fdd -> 9b7c7bd91a)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 1409bc0fdd Performs some minor cleanup of TabletLocator (#3271) add 7ad91eb32a adds missing since tags (#3275) new 9b7c7bd91a Merge remote-tracking branch 'upstream/2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java | 1 + .../java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java | 2 ++ 2 files changed, 3 insertions(+)
[accumulo] branch 2.1 updated: adds missing since tags (#3275)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 7ad91eb32a adds missing since tags (#3275)
7ad91eb32a is described below
commit 7ad91eb32acf7c7c3e0413e24d8526f6cae036ba
Author: Keith Turner
AuthorDate: Tue Apr 4 10:47:16 2023 -0400
adds missing since tags (#3275)
---
core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java | 1 +
.../java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java | 2 ++
2 files changed, 3 insertions(+)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
b/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
index 5270c45221..13bd9c17fa 100644
--- a/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
+++ b/core/src/main/java/org/apache/accumulo/core/client/ScannerBase.java
@@ -50,6 +50,7 @@ public interface ScannerBase extends
Iterable>, AutoCloseable {
* EVENTUAL means that the scanner may not see the latest data that was
written to a TabletServer,
* but may instead see an older version of data.
*
+ * @since 2.1.0
*/
enum ConsistencyLevel {
IMMEDIATE, EVENTUAL
diff --git
a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java
index c3140e86b3..e00ddd3f22 100644
---
a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java
+++
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoop/mapreduce/InputFormatBuilder.java
@@ -232,6 +232,8 @@ public interface InputFormatBuilder {
/**
* Enables the user to set the consistency level
+ *
+ * @since 2.1.0
*/
InputFormatOptions consistencyLevel(ConsistencyLevel level);
}
[accumulo] branch 1.10 updated: Fixes tablet location cache bug that caused batch scanner to return duplicate data (#3168)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 1.10
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/1.10 by this push:
new 7328507396 Fixes tablet location cache bug that caused batch scanner
to return duplicate data (#3168)
7328507396 is described below
commit 73285073969c091db65abe5cfedc2c636c610ece
Author: Keith Turner
AuthorDate: Tue Jan 31 19:29:11 2023 -0500
Fixes tablet location cache bug that caused batch scanner to return
duplicate data (#3168)
---
.../core/client/impl/TabletLocatorImpl.java| 32 +-
.../core/client/impl/TabletLocatorImplTest.java| 119 +
2 files changed, 149 insertions(+), 2 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java
b/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java
index fd76e3b595..0f28bb9587 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/impl/TabletLocatorImpl.java
@@ -275,6 +275,24 @@ public class TabletLocatorImpl extends TabletLocator {
return false;
}
+ static boolean isContiguous(List tabletLocations) {
+
+Iterator iter = tabletLocations.iterator();
+KeyExtent prevExtent = iter.next().tablet_extent;
+
+while (iter.hasNext()) {
+ KeyExtent currExtent = iter.next().tablet_extent;
+
+ if (!currExtent.isPreviousExtent(prevExtent)) {
+return false;
+ }
+
+ prevExtent = currExtent;
+}
+
+return true;
+ }
+
private List binRanges(ClientContext context, List ranges,
Map>> binnedRanges, boolean useCache,
LockCheckerSession lcSession)
@@ -330,8 +348,18 @@ public class TabletLocatorImpl extends TabletLocator {
tabletLocations.add(tl);
}
- for (TabletLocation tl2 : tabletLocations) {
-TabletLocatorImpl.addRange(binnedRanges, tl2.tablet_location,
tl2.tablet_extent, range);
+ // Ensure the extents found are non overlapping and have no holes. When
reading some extents
+ // from the cache and other from the metadata table in the loop above we
may end up with
+ // non-contiguous extents. This can happen when a subset of exents are
placed in the cache and
+ // then after that merges and splits happen.
+ if (isContiguous(tabletLocations)) {
+for (TabletLocation tl2 : tabletLocations) {
+ TabletLocatorImpl.addRange(binnedRanges, tl2.tablet_location,
tl2.tablet_extent, range);
+}
+ } else {
+failures.add(range);
+if (!useCache)
+ lookupFailed = true;
}
}
diff --git
a/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java
b/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java
index 0ba87e5abb..9bf1c07bd2 100644
---
a/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/client/impl/TabletLocatorImplTest.java
@@ -17,6 +17,7 @@
package org.apache.accumulo.core.client.impl;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -1115,6 +1116,124 @@ public class TabletLocatorImplTest {
nrl(nr("0", "11"), nr("1", "2"), nr("0", "4"), nr("2", "4")));
}
+ @Test
+ public void testBinRangesNonContiguousExtents() throws Exception {
+
+// This test exercises a bug that was seen in the tablet locator code.
+
+KeyExtent e1 = nke("foo", "05", null);
+KeyExtent e2 = nke("foo", "1", "05");
+KeyExtent e3 = nke("foo", "2", "05");
+
+Text tableName = new Text("foo");
+
+TServers tservers = new TServers();
+TabletLocatorImpl metaCache =
+createLocators(tservers, "tserver1", "tserver2", "foo", e1, "l1", e2,
"l1");
+
+List ranges = nrl(nr("01", "07"));
+Map>> expected =
+createExpectedBinnings("l1", nol(e1, nrl(nr("01", "07")), e2,
nrl(nr("01", "07";
+
+// The following will result in extents e1 and e2 being placed in the
cache.
+runTest(tableName, ranges, metaCache, expected, nrl());
+
+// Add e3 to the metadata table. Extent e3 could not be added earlier in
the test because it
+// overlaps e2. If e2 and e3 are seen in the same metadata read then one
will be removed from
+// the cache because the cache can never contain overlap
[accumulo] branch 2.1 updated (7f19f96914 -> cb22cc46bb)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch 2.1 in repository https://gitbox.apache.org/repos/asf/accumulo.git from 7f19f96914 Add a last location mode property (#3142) add 7328507396 Fixes tablet location cache bug that caused batch scanner to return duplicate data (#3168) new cb22cc46bb Merge branch '1.10' into 2.1 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/clientImpl/TabletLocatorImpl.java | 32 - .../core/clientImpl/TabletLocatorImplTest.java | 137 - 2 files changed, 166 insertions(+), 3 deletions(-)
[accumulo] 01/01: Merge branch '1.10' into 2.1
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit cb22cc46bb4b8792c5b4c7a7a301b224ca1684c1
Merge: 7f19f96914 7328507396
Author: Keith Turner
AuthorDate: Tue Jan 31 20:21:12 2023 -0500
Merge branch '1.10' into 2.1
.../core/clientImpl/TabletLocatorImpl.java | 32 -
.../core/clientImpl/TabletLocatorImplTest.java | 137 -
2 files changed, 166 insertions(+), 3 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/clientImpl/TabletLocatorImpl.java
index 5b120911bc,00..31ec31c027
mode 100644,00..100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletLocatorImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TabletLocatorImpl.java
@@@ -1,756 -1,0 +1,784 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.clientImpl;
+
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static
org.apache.accumulo.core.util.UtilWaitThread.sleepUninterruptibly;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.TableNotFoundException;
+import org.apache.accumulo.core.data.Key;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.PartialKey;
+import org.apache.accumulo.core.data.Range;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.util.OpTimer;
+import org.apache.accumulo.core.util.Pair;
+import org.apache.accumulo.core.util.TextUtil;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+public class TabletLocatorImpl extends TabletLocator {
+
+ private static final Logger log =
LoggerFactory.getLogger(TabletLocatorImpl.class);
+
+ // MAX_TEXT represents a TEXT object that is greater than all others.
Attempted to use null for
+ // this purpose, but there seems to be a bug in TreeMap.tailMap with null.
Therefore instead of
+ // using null, created MAX_TEXT.
+ static final Text MAX_TEXT = new Text();
+
+ static final Comparator END_ROW_COMPARATOR = (o1, o2) -> {
+if (o1 == o2) {
+ return 0;
+}
+if (o1 == MAX_TEXT) {
+ return 1;
+}
+if (o2 == MAX_TEXT) {
+ return -1;
+}
+return o1.compareTo(o2);
+ };
+
+ protected TableId tableId;
+ protected TabletLocator parent;
+ protected TreeMap metaCache = new
TreeMap<>(END_ROW_COMPARATOR);
+ protected TabletLocationObtainer locationObtainer;
+ private TabletServerLockChecker lockChecker;
+ protected Text lastTabletRow;
+
+ private TreeSet badExtents = new TreeSet<>();
+ private ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock();
+ private final Lock rLock = rwLock.readLock();
+ private final Lock wLock = rwLock.writeLock();
+
+ public interface TabletLocationObtainer {
+/**
+ * @return null when unable to read information successfully
+ */
+TabletLocations lookupTablet(ClientContext context, TabletLocation src,
Text row, Text stopRow,
+TabletLocator parent) throws AccumuloSecurityException,
AccumuloException;
+
+List lookupTablets(ClientContext context, String tserver,
+Map> map, TabletLocator parent)
+
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 8eddbbb4d108e6c30bfa8da237fd5a52ee1a Merge: ab8e2f5188 cb22cc46bb Author: Keith Turner AuthorDate: Tue Jan 31 20:22:24 2023 -0500 Merge branch '2.1' .../core/clientImpl/TabletLocatorImpl.java | 32 - .../core/clientImpl/TabletLocatorImplTest.java | 137 - 2 files changed, 166 insertions(+), 3 deletions(-)
[accumulo] branch main updated (ab8e2f5188 -> 8eddbbb4d1)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from ab8e2f5188 Remove some unused code to get rid of warnings add 7328507396 Fixes tablet location cache bug that caused batch scanner to return duplicate data (#3168) add cb22cc46bb Merge branch '1.10' into 2.1 new 8eddbbb4d1 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/clientImpl/TabletLocatorImpl.java | 32 - .../core/clientImpl/TabletLocatorImplTest.java | 137 - 2 files changed, 166 insertions(+), 3 deletions(-)
[accumulo] branch elasticity updated: Adds support for conditional mutations to Ample (#3251)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 0fc605342c Adds support for conditional mutations to Ample (#3251)
0fc605342c is described below
commit 0fc605342cb615843c73d1c1b2e65399d056ae61
Author: Keith Turner
AuthorDate: Thu Apr 13 14:32:14 2023 -0400
Adds support for conditional mutations to Ample (#3251)
In addition to adding conditional support to Ample, this change also adds
new
operation id metadata column. The operation id is intended to support
running
mutually exclusive tablet operations. The operation id is central to the
goal
of using conditional mutations to allow multiple processes to safely modify
a
tablet and that is why it is being introduced at the same time as general
conditional mutation support.
---
.../core/clientImpl/ConditionalWriterImpl.java | 19 +-
.../accumulo/core/metadata/TabletOperationId.java | 33 +-
.../accumulo/core/metadata/schema/Ample.java | 145 +--
.../core/metadata/schema/MetadataSchema.java | 25 ++
.../core/metadata/schema/RootTabletMetadata.java | 18 +-
.../core/metadata/schema/TabletMetadata.java | 20 +
.../core/metadata/schema/TabletOperation.java | 33 +-
.../server/constraints/MetadataConstraints.java| 3 +-
.../server}/data/ServerConditionalMutation.java| 3 +-
.../metadata/ConditionalTabletMutatorImpl.java | 154
.../metadata/ConditionalTabletsMutatorImpl.java| 117 ++
.../server/metadata/RootConditionalWriter.java | 154
.../server/metadata/RootTabletMutatorImpl.java | 6 +-
.../accumulo/server/metadata/ServerAmpleImpl.java | 5 +
.../server/metadata/TabletMutatorBase.java | 140 ---
.../server/metadata/TabletMutatorImpl.java | 4 +-
.../metadata/iterators/LocationExistsIterator.java | 68
.../metadata/iterators/PresentIterator.java} | 31 +-
.../TabletExistsIterator.java} | 42 +-
.../server/tablets}/ConditionCheckerContext.java | 6 +-
.../manager/state/RootTabletStateStoreTest.java| 27 +-
.../apache/accumulo/manager/TabletOperations.java} | 30 +-
.../accumulo/tserver/ConditionalMutationSet.java | 2 +-
.../java/org/apache/accumulo/tserver/RowLocks.java | 2 +-
.../accumulo/tserver/TabletClientHandler.java | 5 +-
.../org/apache/accumulo/tserver/tablet/Tablet.java | 2 +-
.../test/functional/AmpleConditionalWriterIT.java | 428 +
27 files changed, 1283 insertions(+), 239 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ConditionalWriterImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ConditionalWriterImpl.java
index d029d717c3..9c25135bab 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ConditionalWriterImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ConditionalWriterImpl.java
@@ -89,7 +89,7 @@ import org.apache.thrift.transport.TTransportException;
import com.google.common.net.HostAndPort;
-class ConditionalWriterImpl implements ConditionalWriter {
+public class ConditionalWriterImpl implements ConditionalWriter {
private static final int MAX_SLEEP = 3;
@@ -724,7 +724,7 @@ class ConditionalWriterImpl implements ConditionalWriter {
}
}
- private Status fromThrift(TCMStatus status) {
+ public static Status fromThrift(TCMStatus status) {
switch (status) {
case ACCEPTED:
return Status.ACCEPTED;
@@ -745,12 +745,11 @@ class ConditionalWriterImpl implements ConditionalWriter {
var tcondMutaions = new ArrayList();
for (var cm : mutationList) {
-TMutation tm = cm.toThrift();
+var id = cmid.longValue();
-List conditions = convertConditions(cm, compressedIters);
+TConditionalMutation tcm = convertConditionalMutation(compressedIters,
cm, id);
cmidToCm.put(cmid.longValue(), new CMK(keyExtent, cm));
-TConditionalMutation tcm = new TConditionalMutation(conditions, tm,
cmid.longValue());
cmid.increment();
tcondMutaions.add(tcm);
}
@@ -759,6 +758,14 @@ class ConditionalWriterImpl implements ConditionalWriter {
});
}
+ public static TConditionalMutation
convertConditionalMutation(CompressedIterators compressedIters,
+ ConditionalMutation cm, long id) {
+TMutation tm = cm.toThrift();
+List conditions = convertConditions(cm, compressedIters);
+TConditionalMutation tcm = new TConditionalMutation(conditions, tm, id);
+return tcm;
+ }
+
private static final Comparator TIMESTAMP_COMPARATOR =
Comparator.nullsFirst(Comparator.reverseOrder());
@@ -767,7 +774,7 @@ class ConditionalWriterImpl implements ConditionalWriter
[accumulo] branch elasticity updated: updates tablet locs using conditional mutations (#3317)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 950c806e20 updates tablet locs using conditional mutations (#3317)
950c806e20 is described below
commit 950c806e20abf91d1fc8fc2ececfcf0f935a4d15
Author: Keith Turner
AuthorDate: Tue May 2 18:15:24 2023 -0400
updates tablet locs using conditional mutations (#3317)
Updates tablet locations using conditional mutations. Two supporting changes
were made and one bug was fixes while making this change.
The first supporting change was streamlining handling of conditional
mutations
with a result of UNKNOWN. An UNKNOWN result on a conditional mutation
occurs
when the RPC for the conditional mutation has an error. In this case the
conditional mutation may or may not have gone through. The tablet must be
read
to know what happened. This update adds support for automatically reading
the
tablet and checking it via a lambda. This makes it easy to write code for
handling the unknown case.
The second supporting change was combining code that was mostly the same in
ZooTabletStateStore and MetaDataStateStore by making both extend
AbstractStateStore and use common code. This change allowed the updates to
use
conditional mutations to be made in one place instead of two.
The bug was with the new conditional writer code, it only supported writing
tablets of the same table. The code was changed to only require that
tablets
be on the same datalevel. This change allowed ITs that create multiple
tables
to run.
Some places in the code that set locations were not changed to use
conditional
mutations. Comments were placed in the code for these.
fixes #3284
---
.../accumulo/core/metadata/schema/Ample.java | 43 -
.../manager/state/AbstractTabletStateStore.java| 174 +
.../server/manager/state/MetaDataStateStore.java | 115 +++---
.../server/manager/state/ZooTabletStateStore.java | 74 +++--
.../metadata/ConditionalTabletMutatorImpl.java | 22 ++-
.../metadata/ConditionalTabletsMutatorImpl.java| 101 ++--
.../server/metadata/RootConditionalWriter.java | 3 +-
.../accumulo/server/util/ManagerMetadataUtil.java | 11 +-
.../manager/state/RootTabletStateStoreTest.java| 55 ---
.../ConditionalTabletsMutatorImplTest.java | 140 +
10 files changed, 558 insertions(+), 180 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
index d7a7177da2..c19729ddef 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
@@ -22,6 +22,7 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
import java.util.UUID;
+import java.util.function.Predicate;
import java.util.stream.Stream;
import org.apache.accumulo.core.client.ConditionalWriter;
@@ -246,6 +247,30 @@ public interface Ample {
void close();
}
+ public interface ConditionalResult {
+
+/**
+ * Returns the status of the conditional mutation. If the status was
+ * {@link
org.apache.accumulo.core.client.ConditionalWriter.Status#UNKNOWN} and
+ * Ample#UknownValidator indicates things are ok then this will return
+ * {@link
org.apache.accumulo.core.client.ConditionalWriter.Status#ACCEPTED}
+ */
+ConditionalWriter.Status getStatus();
+
+KeyExtent getExtent();
+
+/**
+ * This can only be called when {@link #getStatus()} returns something
other than
+ * {@link
org.apache.accumulo.core.client.ConditionalWriter.Status#ACCEPTED}. It reads
that
+ * tablets metadata for a failed conditional mutation. This can used used
to see why it failed.
+ * In the case where {@link #getStatus()} returns
+ * {@link
org.apache.accumulo.core.client.ConditionalWriter.Status#UNKNOWN} it can be
used to
+ * see if the mutation actually succeeded or not.
+ *
+ */
+TabletMetadata readMetadata();
+ }
+
public interface ConditionalTabletsMutator extends AutoCloseable {
/**
@@ -260,7 +285,7 @@ public interface Ample {
*
* @return The result from the {@link ConditionalWriter} of processing
each tablet.
*/
-Map process();
+Map process();
@Override
void close();
@@ -371,6 +396,11 @@ public interface Ample {
ConditionalTabletMutator requireAbsentTablet();
}
+ /**
+ * Convenience interface for handling conditional mutations with a status of
UNKNOWN.
+ */
+ interface UnknownValidator extends Predicate {}
+
interface ConditionalTabletMutator
[accumulo] branch 2.1 updated: adds unit test for deferred session cleanup (#3581)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 5282088890 adds unit test for deferred session cleanup (#3581)
5282088890 is described below
commit 5282088890fbb6db4ee2a9ca9bdb9b9c08334ebc
Author: Keith Turner
AuthorDate: Wed Jul 12 12:55:25 2023 -0400
adds unit test for deferred session cleanup (#3581)
---
.../accumulo/tserver/session/SessionManager.java | 6 +-
.../tserver/session/SessionManagerTest.java| 109 +
2 files changed, 114 insertions(+), 1 deletion(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java
index f0f8a5de2a..5041b5c66f 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java
@@ -224,7 +224,7 @@ public class SessionManager {
return session;
}
- private void cleanup(Session session) {
+ static void cleanup(BlockingQueue deferredCleanupQueue, Session
session) {
if (!session.cleanup()) {
var retry = Retry.builder().infiniteRetries().retryAfter(25,
MILLISECONDS)
.incrementBy(25, MILLISECONDS).maxWait(5, SECONDS).backOffFactor(1.5)
@@ -248,6 +248,10 @@ public class SessionManager {
}
}
+ private void cleanup(Session session) {
+cleanup(deferredCleanupQueue, session);
+ }
+
private void sweep(final long maxIdle, final long maxUpdateIdle) {
List sessionsToCleanup = new LinkedList<>();
Iterator iter = sessions.values().iterator();
diff --git
a/server/tserver/src/test/java/org/apache/accumulo/tserver/session/SessionManagerTest.java
b/server/tserver/src/test/java/org/apache/accumulo/tserver/session/SessionManagerTest.java
new file mode 100644
index 00..be3626ea71
--- /dev/null
+++
b/server/tserver/src/test/java/org/apache/accumulo/tserver/session/SessionManagerTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.tserver.session;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.concurrent.ArrayBlockingQueue;
+
+import org.junit.jupiter.api.Test;
+
+public class SessionManagerTest {
+
+ private static class TestSession extends Session {
+
+int cleanupCount;
+
+TestSession(int cleanupCount) {
+ super(null);
+ this.cleanupCount = cleanupCount;
+}
+
+@Override
+public boolean cleanup() {
+ return cleanupCount-- <= 0;
+}
+ }
+
+ @Test
+ public void testTestcode() {
+// test behavior of test class
+TestSession session = new TestSession(2);
+assertFalse(session.cleanup());
+assertFalse(session.cleanup());
+assertTrue(session.cleanup());
+assertTrue(session.cleanup());
+ }
+
+ @Test
+ public void testFullDeferredCleanupQueue() {
+ArrayBlockingQueue deferredCleanupQeue = new
ArrayBlockingQueue<>(3);
+
+deferredCleanupQeue.add(new TestSession(2));
+deferredCleanupQeue.add(new TestSession(2));
+deferredCleanupQeue.add(new TestSession(2));
+
+// the queue is full, so cleanup method should repeatedly call cleanup
instead of queuing
+TestSession session = new TestSession(5);
+SessionManager.cleanup(deferredCleanupQeue, session);
+assertEquals(-1, session.cleanupCount);
+assertEquals(3, deferredCleanupQeue.size());
+assertTrue(deferredCleanupQeue.stream().allMatch(s -> ((TestSession)
s).cleanupCount == 2));
+ }
+
+ @Test
+ public void testDefersCleanup() {
+ArrayBlockingQueue deferredCleanupQeue = new
ArrayBlockingQueue<>(3);
+
+deferredCleanupQeue.add(new TestSession(2));
+deferredCleanupQeue.add(new TestSession(2));
+
+TestSession session = new TestSession(5);
+
+// the queue is
[accumulo] branch main updated (d4846d407e -> 0a3feba20f)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from d4846d407e Merge branch '2.1' add 5282088890 adds unit test for deferred session cleanup (#3581) add 0733d3ae86 adds logging when tablet close is waiting on a scan (#3582) new 0a3feba20f Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../accumulo/tserver/session/SessionManager.java | 6 +- .../accumulo/tserver/tablet/ScanDataSource.java| 11 +++ .../org/apache/accumulo/tserver/tablet/Tablet.java | 11 +++ .../tserver/session/SessionManagerTest.java| 109 + 4 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 server/tserver/src/test/java/org/apache/accumulo/tserver/session/SessionManagerTest.java
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 0a3feba20f2a0d0d542748776777969ba9dcd9b1 Merge: d4846d407e 0733d3ae86 Author: Keith Turner AuthorDate: Wed Jul 12 12:59:36 2023 -0400 Merge branch '2.1' .../accumulo/tserver/session/SessionManager.java | 6 +- .../accumulo/tserver/tablet/ScanDataSource.java| 11 +++ .../org/apache/accumulo/tserver/tablet/Tablet.java | 11 +++ .../tserver/session/SessionManagerTest.java| 109 + 4 files changed, 136 insertions(+), 1 deletion(-)
[accumulo] branch 2.1 updated: adds logging when tablet close is waiting on a scan (#3582)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 0733d3ae86 adds logging when tablet close is waiting on a scan (#3582)
0733d3ae86 is described below
commit 0733d3ae86cae082faab9b40b46bb55149e0852f
Author: Keith Turner
AuthorDate: Wed Jul 12 12:58:05 2023 -0400
adds logging when tablet close is waiting on a scan (#3582)
---
.../org/apache/accumulo/tserver/tablet/ScanDataSource.java| 11 +++
.../main/java/org/apache/accumulo/tserver/tablet/Tablet.java | 11 +++
2 files changed, 22 insertions(+)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanDataSource.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanDataSource.java
index 93b0e8a7f2..51ada87625 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanDataSource.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/ScanDataSource.java
@@ -50,6 +50,8 @@ import
org.apache.accumulo.server.iterators.TabletIteratorEnvironment;
import org.apache.accumulo.tserver.InMemoryMap.MemoryIterator;
import org.apache.accumulo.tserver.TabletServer;
import org.apache.accumulo.tserver.scan.ScanParameters;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -286,4 +288,13 @@ class ScanDataSource implements DataSource {
throw new UnsupportedOperationException();
}
+ @Override
+ public String toString() {
+return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
+.append("isNull(memIters)", memIters == null)
+.append("isNull(fileManager)", fileManager == null)
+.append("fileReservationId",
fileReservationId).append("interruptFlag", interruptFlag.get())
+.append("expectedDeletionCount",
expectedDeletionCount).append("scanParams", scanParams)
+.toString();
+ }
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 8097c3660b..23cc377bcf 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -978,8 +978,19 @@ public class Tablet extends TabletBase {
activeScan.interrupt();
}
+long lastLogTime = System.nanoTime();
+
// wait for reads and writes to complete
while (writesInProgress > 0 || !activeScans.isEmpty()) {
+
+ if (log.isDebugEnabled() && System.nanoTime() - lastLogTime >
TimeUnit.SECONDS.toNanos(60)) {
+for (ScanDataSource activeScan : activeScans) {
+ log.debug("Waiting on scan in completeClose {} {}", extent,
activeScan);
+}
+
+lastLogTime = System.nanoTime();
+ }
+
try {
log.debug("Waiting to completeClose for {}. {} writes {} scans",
extent, writesInProgress,
activeScans.size());
[accumulo] branch elasticity updated: Removed unused code from tablet server. (#3618)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 037dba45cb Removed unused code from tablet server. (#3618)
037dba45cb is described below
commit 037dba45cb606496fb19a10448ea08ddb7717a54
Author: Keith Turner
AuthorDate: Mon Jul 17 15:02:03 2023 -0400
Removed unused code from tablet server. (#3618)
Follow up to #3604. Used IDE to find more unused code.
---
.../org/apache/accumulo/tserver/TabletServer.java | 4 --
.../tserver/TabletServerResourceManager.java | 9
.../apache/accumulo/tserver/TabletStatsKeeper.java | 43 ++---
.../tserver/metrics/TabletServerMetricsUtil.java | 4 --
.../tserver/metrics/TabletServerScanMetrics.java | 4 --
.../accumulo/tserver/tablet/DatafileManager.java | 14 --
.../accumulo/tserver/tablet/SplitRowSpec.java | 31
.../org/apache/accumulo/tserver/tablet/Tablet.java | 56 --
8 files changed, 3 insertions(+), 162 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
index 8b3f55e15c..b1818db63d 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java
@@ -1089,10 +1089,6 @@ public class TabletServer extends AbstractServer
implements TabletHostingServer
return resourceManager.holdTime();
}
- public SecurityOperation getSecurityOperation() {
-return security;
- }
-
// avoid unnecessary redundant markings to meta
final ConcurrentHashMap> metadataTableLogs =
new ConcurrentHashMap<>();
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
index ce90527c4f..58db890afe 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServerResourceManager.java
@@ -650,11 +650,6 @@ public class TabletServerResourceManager {
}
// BEGIN methods that Tablets call to manage their set of open data files
-
-public void importedDataFiles() {
- lastReportedCommitTime = System.currentTimeMillis();
-}
-
public synchronized ScanFileManager newScanFileManager(ScanDispatch
scanDispatch) {
if (closed) {
throw new IllegalStateException("closed");
@@ -733,10 +728,6 @@ public class TabletServerResourceManager {
}
}
}
-
-public TabletServerResourceManager getTabletServerResourceManager() {
- return TabletServerResourceManager.this;
-}
}
public void executeReadAhead(KeyExtent tablet, ScanDispatcher dispatcher,
ScanSession scanInfo,
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletStatsKeeper.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletStatsKeeper.java
index 9fe303bcf9..fb2838b425 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletStatsKeeper.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletStatsKeeper.java
@@ -25,16 +25,14 @@ import org.apache.accumulo.server.util.ActionStatsUpdator;
public class TabletStatsKeeper {
// suspect we need more synchronization in this class
- private ActionStats major = new ActionStats();
private ActionStats minor = new ActionStats();
- private ActionStats split = new ActionStats();
public enum Operation {
// TODO delete split
-MAJOR, SPLIT, MINOR
+MINOR
}
- private ActionStats[] map = {major, split, minor};
+ private ActionStats[] map = {minor};
public void updateTime(Operation operation, long queued, long start, long
count, boolean failed) {
try {
@@ -63,37 +61,11 @@ public class TabletStatsKeeper {
}
- public void updateTime(Operation operation, long start, boolean failed) {
-try {
- ActionStats data = map[operation.ordinal()];
- if (failed) {
-data.fail++;
-data.status--;
- } else {
-double t = (System.currentTimeMillis() - start) / 1000.0;
-
-data.status--;
-data.num++;
-data.elapsed += t;
-data.sumDev += t * t;
-
-if (data.elapsed < 0 || data.sumDev < 0 || data.queueSumDev < 0 ||
data.queueTime < 0) {
- resetTimes();
-}
- }
-} catch (Exception E) {
- resetTimes();
-}
-
- }
-
public void saveMajorMinorTimes(TabletStats t) {
ActionStatsUpdator.update(minor, t.minors);
}
private void resetTimes() {
-major = new ActionStats();
-split = new
[accumulo] branch main updated (27b2113ab7 -> 8456f0597b)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from 27b2113ab7 Merge branch '2.1'
add 688e4ed68b Avoids reusing rfile objects when runtime exception happens
(#3623)
add 8e823f53c5 syncs ZK before reading root tablet metadata (#3601)
add c679b04313 Avoid caching a snapshot of custom props (#3588)
new 8456f0597b Merge remote-tracking branch 'upstream/2.1'
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../core/metadata/schema/TabletsMetadata.java | 6 ++-
.../accumulo/core/util/ConfigurationImpl.java | 24 +---
.../ConfigurationImplTest.java}| 34 -
.../accumulo/server/ServiceEnvironmentImpl.java| 13 +--
.../server/ServiceEnvironmentImplTest.java | 43 +++---
.../apache/accumulo/tserver/tablet/Scanner.java| 27 --
.../org/apache/accumulo/tserver/tablet/Tablet.java | 9 +++--
.../apache/accumulo/tserver/tablet/TabletBase.java | 9 +++--
8 files changed, 88 insertions(+), 77 deletions(-)
copy
core/src/test/java/org/apache/accumulo/core/{data/constraints/NoDeleteConstraintTest.java
=> util/ConfigurationImplTest.java} (55%)
[accumulo] 01/01: Merge remote-tracking branch 'upstream/2.1'
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 8456f0597bf81d2b79537066a6f050614a7411b7
Merge: 27b2113ab7 c679b04313
Author: Keith Turner
AuthorDate: Tue Jul 18 12:08:22 2023 -0400
Merge remote-tracking branch 'upstream/2.1'
.../core/metadata/schema/TabletsMetadata.java | 6 ++-
.../accumulo/core/util/ConfigurationImpl.java | 24 +--
.../accumulo/core/util/ConfigurationImplTest.java | 49 ++
.../accumulo/server/ServiceEnvironmentImpl.java| 13 --
.../server/ServiceEnvironmentImplTest.java | 43 +--
.../apache/accumulo/tserver/tablet/Scanner.java| 27 ++--
.../org/apache/accumulo/tserver/tablet/Tablet.java | 9 ++--
.../apache/accumulo/tserver/tablet/TabletBase.java | 9 ++--
8 files changed, 120 insertions(+), 60 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
index 7b1a61a8d6,600e889f5e..f431cb052c
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
@@@ -574,10 -537,14 +574,14 @@@ public class TabletsMetadata implement
case IMMEDIATE:
ZooReader zooReader = ctx.getZooReader();
try {
- byte[] bytes = zooReader.getData(zkRoot + RootTable.ZROOT_TABLET);
+ var path = zkRoot + RootTable.ZROOT_TABLET;
+ // attempt (see ZOOKEEPER-1675) to ensure the latest root table
metadata is read from
+ // zookeeper
+ zooReader.sync(path);
+ byte[] bytes = zooReader.getData(path);
return new RootTabletMetadata(new String(bytes,
UTF_8)).toTabletMetadata();
} catch (InterruptedException | KeeperException e) {
- throw new RuntimeException(e);
+ throw new IllegalStateException(e);
}
default:
throw new IllegalArgumentException("Unknown consistency level " +
readConsistency);
diff --cc
server/base/src/main/java/org/apache/accumulo/server/ServiceEnvironmentImpl.java
index 9181f8350d,0f68a11d48..46c6fdb7bd
---
a/server/base/src/main/java/org/apache/accumulo/server/ServiceEnvironmentImpl.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/ServiceEnvironmentImpl.java
@@@ -18,8 -18,8 +18,7 @@@
*/
package org.apache.accumulo.server;
- import java.util.Map;
- import java.util.concurrent.ConcurrentHashMap;
-import java.io.IOException;
+ import java.util.concurrent.TimeUnit;
import org.apache.accumulo.core.classloader.ClassLoaderUtil;
import org.apache.accumulo.core.client.TableNotFoundException;
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
index 5092984d92,311492b822..0e02ffce3f
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
@@@ -148,19 -147,23 +147,23 @@@ public class Scanner
sawException = true;
throw re;
} finally {
- // code in finally block because always want
- // to return datafiles, even when exception is thrown
- if (dataSource != null && !scanParams.isIsolated()) {
- dataSource.close(false);
- } else if (dataSource != null) {
- dataSource.detachFileManager();
- }
+ try {
+ // code in finally block because always want
-// to return mapfiles, even when exception is thrown
++// to return datafiles, even when exception is thrown
+ if (dataSource != null) {
+ if (sawException || !scanParams.isIsolated()) {
+ dataSource.close(sawException);
+ } else {
+ dataSource.detachFileManager();
+ }
+ }
- if (results != null && results.getResults() != null) {
- tablet.updateQueryStats(results.getResults().size(),
results.getNumBytes());
+ if (results != null && results.getResults() != null) {
+ tablet.updateQueryStats(results.getResults().size(),
results.getNumBytes());
+ }
+ } finally {
+ scannerSemaphore.release();
}
-
- scannerSemaphore.release();
}
}
diff --cc
server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 2d910585c1,251d9df30b..8f3b5d8157
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@@ -440,13 -456,13 +441,13 @@@ public class Tablet extends TabletBase
try {
SortedKeyValueIterator iter = new
SourceSwitchingIterator(dataSource);
checker.check(iter);
- } catch (IOException ioe) {
- dataSource.close(true);
- thr
[accumulo] branch 2.1 updated: Avoid caching a snapshot of custom props (#3588)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new c679b04313 Avoid caching a snapshot of custom props (#3588)
c679b04313 is described below
commit c679b04313ae4117f290960cca637dd7bc78baf3
Author: Keith Turner
AuthorDate: Tue Jul 18 12:02:39 2023 -0400
Avoid caching a snapshot of custom props (#3588)
ConfigurationImpl which is provided to plugin impl cached a snapshot of
custom props indefinitely. However other props were not cached in this
way. This change removes the indefinited caching of custom props.
---
.../accumulo/core/util/ConfigurationImpl.java | 24 +--
.../accumulo/core/util/ConfigurationImplTest.java | 49 ++
.../accumulo/server/ServiceEnvironmentImpl.java| 13 --
.../server/ServiceEnvironmentImplTest.java | 43 +--
4 files changed, 90 insertions(+), 39 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/util/ConfigurationImpl.java
b/core/src/main/java/org/apache/accumulo/core/util/ConfigurationImpl.java
index 6bb2015618..e80cb998ad 100644
--- a/core/src/main/java/org/apache/accumulo/core/util/ConfigurationImpl.java
+++ b/core/src/main/java/org/apache/accumulo/core/util/ConfigurationImpl.java
@@ -38,11 +38,15 @@ import
org.apache.accumulo.core.spi.common.ServiceEnvironment.Configuration;
public class ConfigurationImpl implements Configuration {
private final AccumuloConfiguration acfg;
- private Map customProps;
- private Map tableCustomProps;
+ private final AccumuloConfiguration.Deriver>
tableCustomDeriver;
+ private final AccumuloConfiguration.Deriver>
customDeriver;
public ConfigurationImpl(AccumuloConfiguration acfg) {
this.acfg = acfg;
+this.customDeriver =
+acfg.newDeriver(aconf -> buildCustom(aconf,
Property.GENERAL_ARBITRARY_PROP_PREFIX));
+this.tableCustomDeriver =
+acfg.newDeriver(aconf -> buildCustom(aconf,
Property.TABLE_ARBITRARY_PROP_PREFIX));
}
@Override
@@ -83,11 +87,7 @@ public class ConfigurationImpl implements Configuration {
@Override
public Map getCustom() {
-if (customProps == null) {
- customProps = buildCustom(Property.GENERAL_ARBITRARY_PROP_PREFIX);
-}
-
-return customProps;
+return customDeriver.derive();
}
@Override
@@ -97,11 +97,7 @@ public class ConfigurationImpl implements Configuration {
@Override
public Map getTableCustom() {
-if (tableCustomProps == null) {
- tableCustomProps = buildCustom(Property.TABLE_ARBITRARY_PROP_PREFIX);
-}
-
-return tableCustomProps;
+return tableCustomDeriver.derive();
}
@Override
@@ -109,8 +105,8 @@ public class ConfigurationImpl implements Configuration {
return getTableCustom().get(keySuffix);
}
- private Map buildCustom(Property customPrefix) {
-return
acfg.getAllPropertiesWithPrefix(customPrefix).entrySet().stream().collect(
+ private static Map buildCustom(AccumuloConfiguration conf,
Property customPrefix) {
+return
conf.getAllPropertiesWithPrefix(customPrefix).entrySet().stream().collect(
Collectors.toUnmodifiableMap(e ->
e.getKey().substring(customPrefix.getKey().length()),
Entry::getValue));
}
diff --git
a/core/src/test/java/org/apache/accumulo/core/util/ConfigurationImplTest.java
b/core/src/test/java/org/apache/accumulo/core/util/ConfigurationImplTest.java
new file mode 100644
index 00..fba1bee4a0
--- /dev/null
+++
b/core/src/test/java/org/apache/accumulo/core/util/ConfigurationImplTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.core.util;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+import java.util.Map;
+
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.junit.jupiter.api.Test;
+
+public class ConfigurationImplTest {
+ @Test
+ public void testCustomProps() {
+Configurati
[accumulo] branch 2.1 updated: Avoids reusing rfile objects when runtime exception happens (#3623)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 688e4ed68b Avoids reusing rfile objects when runtime exception happens
(#3623)
688e4ed68b is described below
commit 688e4ed68b4980eccd77931fee3996987018fe5c
Author: Keith Turner
AuthorDate: Tue Jul 18 11:24:59 2023 -0400
Avoids reusing rfile objects when runtime exception happens (#3623)
Rfiles were not reused when IOException happened, but were reused when a
runtime exception happened. This changes the code to not reuse when a
runtime exception happens.
See #3617
---
.../apache/accumulo/tserver/tablet/Scanner.java| 27 --
.../org/apache/accumulo/tserver/tablet/Tablet.java | 9
.../apache/accumulo/tserver/tablet/TabletBase.java | 9
3 files changed, 25 insertions(+), 20 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
index 107d36f24a..311492b822 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Scanner.java
@@ -137,7 +137,6 @@ public class Scanner {
}
sawException = true;
- dataSource.close(true);
throw ioe;
} catch (RuntimeException re) {
if (ShutdownUtil.wasCausedByHadoopShutdown(re)) {
@@ -148,19 +147,23 @@ public class Scanner {
sawException = true;
throw re;
} finally {
- // code in finally block because always want
- // to return mapfiles, even when exception is thrown
- if (dataSource != null && !scanParams.isIsolated()) {
-dataSource.close(false);
- } else if (dataSource != null) {
-dataSource.detachFileManager();
- }
+ try {
+// code in finally block because always want
+// to return mapfiles, even when exception is thrown
+if (dataSource != null) {
+ if (sawException || !scanParams.isIsolated()) {
+dataSource.close(sawException);
+ } else {
+dataSource.detachFileManager();
+ }
+}
- if (results != null && results.getResults() != null) {
-tablet.updateQueryStats(results.getResults().size(),
results.getNumBytes());
+if (results != null && results.getResults() != null) {
+ tablet.updateQueryStats(results.getResults().size(),
results.getNumBytes());
+}
+ } finally {
+scannerSemaphore.release();
}
-
- scannerSemaphore.release();
}
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 23cc377bcf..251d9df30b 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -452,16 +452,17 @@ public class Tablet extends TabletBase {
ScanDataSource dataSource = createDataSource(scanParams, false, iFlag);
+boolean sawException = false;
try {
SortedKeyValueIterator iter = new
SourceSwitchingIterator(dataSource);
checker.check(iter);
-} catch (IOException ioe) {
- dataSource.close(true);
- throw ioe;
+} catch (IOException | RuntimeException e) {
+ sawException = true;
+ throw e;
} finally {
// code in finally block because always want
// to return mapfiles, even when exception is thrown
- dataSource.close(false);
+ dataSource.close(sawException);
}
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
index ce8cddc348..12890106d4 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java
@@ -201,19 +201,20 @@ public abstract class TabletBase {
Tablet.LookupResult result = null;
+boolean sawException = false;
try {
SortedKeyValueIterator iter = new
SourceSwitchingIterator(dataSource);
this.lookupCount.incrementAndGet();
this.server.getScanMetrics().incrementLookupCount(1);
result = lookup(iter, ranges, results, scanParams, maxResultSize);
return result;
-} catch (IOException ioe) {
- dataSource.close(true);
- throw ioe;
+} catch (IOException | RuntimeException e) {
+ sawException = true;
+ throw e;
} finally {
// code in finally block because always want
// to return mapfiles, even
[accumulo] branch 2.1 updated: syncs ZK before reading root tablet metadata (#3601)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 8e823f53c5 syncs ZK before reading root tablet metadata (#3601)
8e823f53c5 is described below
commit 8e823f53c5d191d9b16da43e12183305d3b58bd0
Author: Keith Turner
AuthorDate: Tue Jul 18 11:25:41 2023 -0400
syncs ZK before reading root tablet metadata (#3601)
---
.../org/apache/accumulo/core/metadata/schema/TabletsMetadata.java | 6 +-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
index f6042f184b..600e889f5e 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletsMetadata.java
@@ -537,7 +537,11 @@ public class TabletsMetadata implements
Iterable, AutoCloseable
case IMMEDIATE:
ZooReader zooReader = ctx.getZooReader();
try {
- byte[] bytes = zooReader.getData(zkRoot + RootTable.ZROOT_TABLET);
+ var path = zkRoot + RootTable.ZROOT_TABLET;
+ // attempt (see ZOOKEEPER-1675) to ensure the latest root table
metadata is read from
+ // zookeeper
+ zooReader.sync(path);
+ byte[] bytes = zooReader.getData(path);
return new RootTabletMetadata(new String(bytes,
UTF_8)).toTabletMetadata();
} catch (InterruptedException | KeeperException e) {
throw new RuntimeException(e);
[accumulo] branch no-chop-merge updated: adds range validation to tablet file objects
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch no-chop-merge
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/no-chop-merge by this push:
new 883d3d8b32 adds range validation to tablet file objects
883d3d8b32 is described below
commit 883d3d8b32faf839b98b6f3a604bc0d3f8813302
Author: Keith Turner
AuthorDate: Fri Jul 14 21:02:00 2023 -0400
adds range validation to tablet file objects
---
.../core/metadata/ReferencedTabletFile.java| 3 ++-
.../accumulo/core/metadata/StoredTabletFile.java | 23 +-
.../metadata/schema/ReferencedTabletFileTest.java | 23 ++
3 files changed, 47 insertions(+), 2 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java
b/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java
index 0300f0ee31..bd5af63091 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/ReferencedTabletFile.java
@@ -19,6 +19,7 @@
package org.apache.accumulo.core.metadata;
import static org.apache.accumulo.core.Constants.HDFS_TABLES_DIR;
+import static
org.apache.accumulo.core.metadata.StoredTabletFile.requireRowRange;
import java.net.URI;
import java.util.Comparator;
@@ -165,7 +166,7 @@ public class ReferencedTabletFile extends
AbstractTabletFile {
return new StoredTabletFile(metadataEntry);
}
+ private static boolean isOnlyRowSet(Key key) {
+return key.getColumnFamilyData().length() == 0 &&
key.getColumnQualifierData().length() == 0
+&& key.getColumnVisibilityData().length() == 0 && key.getTimestamp()
== Long.MAX_VALUE;
+ }
+
+ static Range requireRowRange(Range range) {
+if (!range.isInfiniteStartKey()) {
+ Preconditions.checkArgument(range.isStartKeyInclusive() &&
isOnlyRowSet(range.getStartKey()),
+ "Range is not a row range %s", range);
+}
+
+if (!range.isInfiniteStopKey()) {
+ Preconditions.checkArgument(!range.isEndKeyInclusive() &&
isOnlyRowSet(range.getEndKey()),
+ "Range is not a row range %s", range);
+}
+
+return range;
+ }
+
public static StoredTabletFile of(final URI path, Range range) {
return of(new Path(Objects.requireNonNull(path)), range);
}
public static StoredTabletFile of(final Path path, Range range) {
-return new StoredTabletFile(new TabletFileCq(Objects.requireNonNull(path),
range));
+return new StoredTabletFile(
+new TabletFileCq(Objects.requireNonNull(path),
requireRowRange(range)));
}
private static final Gson gson =
ByteArrayToBase64TypeAdapter.createBase64Gson();
diff --git
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/ReferencedTabletFileTest.java
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/ReferencedTabletFileTest.java
index a71f26365c..ff4b818926 100644
---
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/ReferencedTabletFileTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/ReferencedTabletFileTest.java
@@ -24,10 +24,12 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import java.net.URI;
+import org.apache.accumulo.core.data.Key;
import org.apache.accumulo.core.data.Range;
import org.apache.accumulo.core.data.TableId;
import org.apache.accumulo.core.metadata.ReferencedTabletFile;
import org.apache.accumulo.core.metadata.StoredTabletFile;
+import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.Test;
public class ReferencedTabletFileTest {
@@ -117,4 +119,25 @@ public class ReferencedTabletFileTest {
assertEquals(niceFile.hashCode(), uglyFile.hashCode());
}
+ @Test
+ public void testNonRowRange() {
+Path testPath = new
Path("hdfs://localhost:8020/accumulo/tables/2a/default_tablet/F070.rf");
+
+// range where start key is not a row key
+Range r1 = new Range(new Key("r1", "f1"), true, null, false);
+assertThrows(IllegalArgumentException.class, () -> new
ReferencedTabletFile(testPath, r1));
+
+// range where end key is not a row key
+Range r2 = new Range(null, true, new Key("r1", "f1"), false);
+assertThrows(IllegalArgumentException.class, () -> new
ReferencedTabletFile(testPath, r2));
+
+// range where the key looks like a row, but the start key inclusivity is
not whats expected
+Range r3 = new Range(new Key("r1"), false, new Key("r2"), false);
+assertThrows(IllegalArgumentException.class, () -> new
ReferencedTabletFile(testPath, r3));
+
+// range where the key looks like a row, but the end key inclusivity is
not whats expected
+Range r4 = new Range(new Key("r1"), true, new Key("r2"), true);
+assertThrows(IllegalArgumentException.class, () -> new
ReferencedTabletFile(testPath, r4));
+ }
+
}
[accumulo] branch 2.1 updated: tests custom prop changes are seen (#3627)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new abd7c6c2f0 tests custom prop changes are seen (#3627)
abd7c6c2f0 is described below
commit abd7c6c2f0b6e49e467afdb859d024b573e58a7c
Author: Keith Turner
AuthorDate: Wed Jul 19 12:46:38 2023 -0400
tests custom prop changes are seen (#3627)
This is a follow up to #3588. Realized that ConfigurationCopy supported
the Deriver that ConfigrationImpl uses to detect changes, so added unit
test that leverage this.
---
.../apache/accumulo/core/util/ConfigurationImplTest.java | 16
1 file changed, 16 insertions(+)
diff --git
a/core/src/test/java/org/apache/accumulo/core/util/ConfigurationImplTest.java
b/core/src/test/java/org/apache/accumulo/core/util/ConfigurationImplTest.java
index fba1bee4a0..dbc0407b21 100644
---
a/core/src/test/java/org/apache/accumulo/core/util/ConfigurationImplTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/util/ConfigurationImplTest.java
@@ -45,5 +45,21 @@ public class ConfigurationImplTest {
assertNull(confImp.getCustom("p1"));
assertNull(confImp.getTableCustom("p3"));
+
+// ensure changes to custom props are seen
+conf.set("general.custom.p4", "v5");
+conf.set("table.custom.p2", "v6");
+conf.set("table.custom.p5", "v7");
+
+assertEquals(Map.of("p3", "v3", "p4", "v5"), confImp.getCustom());
+assertEquals(Map.of("p1", "v1", "p2", "v6", "p5", "v7"),
confImp.getTableCustom());
+
+assertEquals("v3", confImp.getCustom("p3"));
+assertEquals("v5", confImp.getCustom("p4"));
+assertEquals("v1", confImp.getTableCustom("p1"));
+assertEquals("v6", confImp.getTableCustom("p2"));
+
+assertNull(confImp.getCustom("p5"));
+assertNull(confImp.getTableCustom("p4"));
}
}
[accumulo] branch elasticity updated: Add Metrics for CompactionJobPriority Queues (#3551)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 6e745d16fa Add Metrics for CompactionJobPriority Queues (#3551)
6e745d16fa is described below
commit 6e745d16fa40524b60f63adc2873ab1f42b2454a
Author: Daniel Roberts
AuthorDate: Wed Jul 19 22:38:35 2023 -0400
Add Metrics for CompactionJobPriority Queues (#3551)
Adds the following metrics for CompactionJobPriority:
* Number of Queues
* Queue Size
* Number of Queued Jobs
* Number of Dequeued Jobs
* Number of Rejected Jobs
* Lowest Job Priority Metric
Other changes in commit
* Adds configurable property to change CompactionQueue size and test
rejectedJobs. See #3635 for more details.
* Switches metric names to dot notation
* Adds CompactionPriorityQueueMetricsIT
* Adds User defined CompactionResourceGroup override to MAC
* Adds configurable property to change CompactionQueue size and test
rejectedJobs. See #3635 for more details.
---
.../org/apache/accumulo/core/conf/Property.java| 3 +
.../accumulo/core/metrics/MetricsProducer.java | 55 ++-
.../apache/accumulo/core/metrics/MetricsUtil.java | 30 ++
.../accumulo/core/metrics/MetricsUtilTest.java | 43 +++
.../miniclusterImpl/MiniAccumuloClusterImpl.java | 5 +-
.../miniclusterImpl/MiniAccumuloConfigImpl.java| 2 +
.../java/org/apache/accumulo/manager/Manager.java | 3 +-
.../queue/CompactionJobPriorityQueue.java | 28 +-
.../compaction/queue/CompactionJobQueues.java | 46 ++-
.../accumulo/manager/metrics/ManagerMetrics.java | 3 +
.../accumulo/manager/metrics/QueueMetrics.java | 100 ++
.../CompactionPriorityQueueMetricsIT.java | 397 +
12 files changed, 707 insertions(+), 8 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 3fca886afc..8cdf429882 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -373,6 +373,9 @@ public enum Property {
"1.10.0"),
MANAGER_SPLIT_WORKER_THREADS("manager.split.inspection.threadpool.size",
"8", PropertyType.COUNT,
"The number of threads used to inspect tablets files to find split
points.", "4.0.0"),
+
+
MANAGER_COMPACTION_SERVICE_PRIORITY_QUEUE_SIZE("manager.compaction.major.service.queue.size",
+ "1", PropertyType.COUNT, "The max size of the priority queue",
"4.0"),
// properties that are specific to scan server behavior
@Experimental
SSERV_PREFIX("sserver.", null, PropertyType.PREFIX,
diff --git
a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
index 322b236e58..820a432bd2 100644
--- a/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
+++ b/core/src/main/java/org/apache/accumulo/core/metrics/MetricsProducer.java
@@ -48,7 +48,7 @@ import io.micrometer.core.instrument.MeterRegistry;
* N/A
* N/A
* {@link #METRICS_LOW_MEMORY}
- * Guage
+ * Gauge
* reports 1 when process memory usage is above threshold, 0 when memory
is okay
*
*
@@ -59,6 +59,48 @@ import io.micrometer.core.instrument.MeterRegistry;
*
*
*
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUES}
+ * Gauge
+ *
+ *
+ *
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_LENGTH}
+ * Gauge
+ *
+ *
+ *
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_PRIORITY}
+ * Gauge
+ *
+ *
+ *
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_QUEUED}
+ * Gauge
+ *
+ *
+ *
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_DEQUEUED}
+ * Gauge
+ *
+ *
+ *
+ * N/A
+ * N/A
+ * {@link #METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_REJECTED}
+ * Gauge
+ *
+ *
+ *
* currentFateOps
* Gauge
* {@link #METRICS_FATE_TOTAL_IN_PROGRESS}
@@ -597,6 +639,17 @@ public interface MetricsProducer {
String METRICS_LOW_MEMORY = "accumulo.detected.low.memory";
String METRICS_COMPACTOR_PREFIX = "accumulo.compactor.";
String METRICS_COMPACTOR_MAJC_STUCK = METRICS_COMPACTOR_PREFIX +
"majc.stuck";
+ String METRICS_COMPACTOR_JOB_PRIORITY_QUEUES = METRICS_COMPACTOR_PREFIX +
"queue.count";
+ String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_LENGTH =
METRICS_COMPACTOR_PREFIX + "queue.length";
+ String METRICS_COMPACTOR_JOB_PRIORITY_QUEUE_JOBS_DEQUEUED =
+ METRICS_COMPACTOR_PREFIX + "queue.jobs.dequeued";
+ String METRIC
[accumulo] branch elasticity updated: pre load ondemand tablets for linear scans (#3429)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 78dc351005 pre load ondemand tablets for linear scans (#3429)
78dc351005 is described below
commit 78dc3510057bbe8b90f4829b2df8b51058b301f7
Author: Keith Turner
AuthorDate: Wed Jul 5 16:35:59 2023 -0400
pre load ondemand tablets for linear scans (#3429)
---
.../core/clientImpl/ClientTabletCache.java | 36 +++-
.../core/clientImpl/ClientTabletCacheImpl.java | 71 ++--
.../core/clientImpl/RootClientTabletCache.java | 2 +-
.../core/clientImpl/SyncingClientTabletCache.java | 8 +-
.../accumulo/core/clientImpl/ThriftScanner.java| 97 +-
.../core/clientImpl/TimeoutClientTabletCache.java | 8 +-
.../apache/accumulo/test/functional/SplitIT.java | 6 --
7 files changed, 204 insertions(+), 24 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java
index fb8c4f7a8c..594e2f85bd 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java
@@ -75,6 +75,36 @@ public abstract class ClientTabletCache {
REQUIRED, NOT_REQUIRED
}
+ /**
+ * This method allows linear scans to host tablet ahead of time that they
may read in the future.
+ * The goal of this method is to allow tablets to request hosting of tablet
for a scan before the
+ * scan actually needs it. Below is an example of how this method could work
with a scan when
+ * {@code minimumHostAhead=4} is passed and avoid the scan having to wait on
tablet hosting.
+ *
+ *
+ * 4*2 tablets are initially hosted (the scan has to wait on this)
+ * The 1st,2nd,3rd, and 4th tablets are read by the scan
+ * The request to read the 5th tablets causes a request to host 4 more
tablets (this would be
+ * the 9th,10th,11th, and 12th tablets)
+ * The 5th,6th,7th, and 8th tablet are read by the scan
+ * While the scan does the read above, the 9th,10th,11th, and 12th
tablets are actually
+ * hosted. This happens concurrently with the scan above.
+ * When the scan goes to read the 9th tablet, hopefully its already
hosted. Also attempting to
+ * read the 9th tablet will cause a request to host the 13th,14th,15th, and
16th tablets.
+ *
+ *
+ * In the situation above, the goal is that while we are reading 4 hosted
tablets the 4 following
+ * tablets are in the process of being hosted.
+ *
+ * @param minimumHostAhead Attempts to keep between minimumHostAhead and
2*minimumHostAhead
+ *tablets following the found tablet hosted.
+ * @param hostAheadRange Only host following tablets that are within this
range.
+ */
+ public abstract CachedTablet findTablet(ClientContext context, Text row,
boolean skipRow,
+ LocationNeed locationNeed, int minimumHostAhead, Range hostAheadRange)
+ throws AccumuloException, AccumuloSecurityException,
TableNotFoundException,
+ InvalidTabletHostingRequestException;
+
/**
* Finds the tablet that contains the given row.
*
@@ -85,9 +115,11 @@ public abstract class ClientTabletCache {
* @return overlapping tablet. If no overlapping tablet exists, returns
null. If location is
* required and the tablet currently has no location ,returns null.
*/
- public abstract CachedTablet findTablet(ClientContext context, Text row,
boolean skipRow,
+ public CachedTablet findTablet(ClientContext context, Text row, boolean
skipRow,
LocationNeed locationNeed) throws AccumuloException,
AccumuloSecurityException,
- TableNotFoundException, InvalidTabletHostingRequestException;
+ TableNotFoundException, InvalidTabletHostingRequestException {
+return findTablet(context, row, skipRow, locationNeed, 0, null);
+ }
public CachedTablet findTabletWithRetry(ClientContext context, Text row,
boolean skipRow,
LocationNeed locationNeed) throws AccumuloException,
AccumuloSecurityException,
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java
index 5e3e8bb440..85271cac69 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCacheImpl.java
@@ -67,6 +67,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
@@ -535,8 +536,9 @@ public class
[accumulo] branch elasticity updated: fixes bug in tablet mgmt scanner (#3566)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 21c2e4479c fixes bug in tablet mgmt scanner (#3566)
21c2e4479c is described below
commit 21c2e4479cf24bcb94930a404d1ad84fd2e0ce46
Author: Keith Turner
AuthorDate: Wed Jul 5 15:49:46 2023 -0400
fixes bug in tablet mgmt scanner (#3566)
---
.../accumulo/server/manager/state/TabletManagementScanner.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementScanner.java
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementScanner.java
index 63e9cd6cd0..bbd97fe6bf 100644
---
a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementScanner.java
+++
b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletManagementScanner.java
@@ -69,9 +69,9 @@ public class TabletManagementScanner implements
ClosableIterator locations = new HashSet<>();
-final List failures = locator.findTablets(context,
ALL_TABLETS_RANGE, (ct, r) -> {
- locations.add(ct.getTserverLocation().orElseThrow());
-}, LocationNeed.NOT_REQUIRED);
+final List failures = locator.findTablets(context,
ALL_TABLETS_RANGE,
+(ct, r) -> ct.getTserverLocation().ifPresent(locations::add),
+LocationNeed.NOT_REQUIRED);
// If failures is not empty, then there are tablets that we don't know
the location of.
// In this case, add an extra thread.
numLocations = (failures.isEmpty()) ? locations.size() :
locations.size() + 1;
[accumulo] branch elasticity updated (c49ee4c4c4 -> 99f9595c57)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from c49ee4c4c4 Set TabletGroupWatcher scan threads with tablet locs and config (#3545) add 99f9595c57 replaces TabletMetadataImposter with new builder (#3567) No new revisions were added by this update. Summary of changes: .../core/metadata/schema/TabletMetadata.java | 26 +-- .../metadata/schema/TabletMetadataBuilder.java | 5 +- .../core/metadata/schema/TabletMetadataTest.java | 97 +++ .../manager/state/TabletMetadataImposter.java | 66 .../manager/state/TabletMetadataImposterTest.java | 188 - .../gc/GarbageCollectWriteAheadLogsTest.java | 14 +- .../accumulo/tserver/UnloadTabletHandler.java | 9 +- .../apache/accumulo/test/manager/MergeStateIT.java | 10 +- 8 files changed, 134 insertions(+), 281 deletions(-) delete mode 100644 server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletMetadataImposter.java delete mode 100644 server/base/src/test/java/org/apache/accumulo/server/manager/state/TabletMetadataImposterTest.java
[accumulo] branch elasticity updated: Removes external compaction final state metadata (#3565)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new a1f1fb85b2 Removes external compaction final state metadata (#3565)
a1f1fb85b2 is described below
commit a1f1fb85b2e8dd715b04f2676ad83ac035815de8
Author: Keith Turner
AuthorDate: Wed Jul 5 15:03:32 2023 -0400
Removes external compaction final state metadata (#3565)
Removes metadata related to how the external compaction commit
process used to work.
fixes #3465
---
.../accumulo/core/metadata/schema/Ample.java | 14 ---
.../schema/ExternalCompactionFinalState.java | 139 -
.../core/metadata/schema/MetadataSchema.java | 13 --
.../ClusterServerConfiguration.java| 2 +-
.../accumulo/server/metadata/ServerAmpleImpl.java | 52
.../coordinator/DeadCompactionDetector.java| 2 +-
.../accumulo/manager/upgrade/Upgrader11to12.java | 70 ++-
.../accumulo/tserver/tablet/CompactableImpl.java | 8 +-
.../test/compaction/ExternalCompactionTServer.java | 47 ---
.../compaction/ExternalCompactionTestUtils.java| 27 ++--
.../test/compaction/ExternalCompaction_1_IT.java | 136 +++-
.../test/compaction/ExternalCompaction_2_IT.java | 95 --
12 files changed, 111 insertions(+), 494 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
index 78a047dabe..082296446a 100644
--- a/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
+++ b/core/src/main/java/org/apache/accumulo/core/metadata/schema/Ample.java
@@ -214,20 +214,6 @@ public interface Ample {
throw new UnsupportedOperationException();
}
- default void
-
putExternalCompactionFinalStates(Collection
finalStates) {
-throw new UnsupportedOperationException();
- }
-
- default Stream
getExternalCompactionFinalStates() {
-throw new UnsupportedOperationException();
- }
-
- default void
- deleteExternalCompactionFinalStates(Collection
statusesToDelete) {
-throw new UnsupportedOperationException();
- }
-
/**
* Return an encoded delete marker Mutation to delete the specified
TabletFile path. A
* ReferenceFile is used for the parameter because the Garbage Collector is
optimized to store a
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionFinalState.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionFinalState.java
deleted file mode 100644
index 907982f2ed..00
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/ExternalCompactionFinalState.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * https://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.accumulo.core.metadata.schema;
-
-import static org.apache.accumulo.core.util.LazySingletons.GSON;
-
-import java.util.Base64;
-
-import org.apache.accumulo.core.data.TableId;
-import org.apache.accumulo.core.dataImpl.KeyExtent;
-import org.apache.accumulo.core.util.TextUtil;
-import org.apache.hadoop.io.Text;
-
-import com.google.common.base.Preconditions;
-
-// ELASTICITY_TODO remove this class, remove it from ample, add upgrade code
to remove it from metadata table
-public class ExternalCompactionFinalState {
-
- public enum FinalState {
-FINISHED, FAILED
- }
-
- private final ExternalCompactionId ecid;
- private final KeyExtent extent;
- private final FinalState state;
- private final long fileSize;
- private final long fileEntries;
-
- public ExternalCompactionFinalState(ExternalCompactionId ecid, KeyExtent
extent, FinalState state,
- long fileSize, long fileEntries) {
-this.ecid = ecid;
-this.extent = extent;
-this.state = state;
-this.fileSize = fileSize;
-this.fileEntries = fileEntries;
- }
-
- public ExternalCompactionId getExternalCompactionId() {
-return ecid;
- }
-
- public FinalState getFinalS
[accumulo] branch 2.1 updated: logs info when a tablet consistency check goes from bad to good (#3574)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 20cecbe8b9 logs info when a tablet consistency check goes from bad to
good (#3574)
20cecbe8b9 is described below
commit 20cecbe8b9c8b55d17f341dfd64f5b42f7f79705
Author: Keith Turner
AuthorDate: Mon Jul 10 11:29:05 2023 -0400
logs info when a tablet consistency check goes from bad to good (#3574)
When the tablet server does a periodic consistency check on a tablet
it logs an error when there is a mismatch. Later if there is no
longer a mismatch, then this commit changes the code to log an info.
related to #3537 but does not fix that issue
---
.../java/org/apache/accumulo/tserver/tablet/Tablet.java | 13 +++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index 489dbd0700..1b818d8759 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -1114,6 +1114,8 @@ public class Tablet extends TabletBase {
}
}
+ private boolean loggedErrorForTabletComparison = false;
+
/**
* Checks that tablet metadata from the metadata table matches what this
tablet has in memory. The
* caller of this method must acquire the updateCounter parameter before
acquiring the
@@ -1162,10 +1164,17 @@ public class Tablet extends TabletBase {
} else {
log.error("Data files in {} differ from in-memory data {} {} {} {}",
extent,
tabletMetadata.getFilesMap(), dataFileSizes, updateCounter,
latestCount);
+loggedErrorForTabletComparison = true;
}
} else {
- log.trace("AMCC Tablet {} files in memory are same as in metadata table
{}",
- tabletMetadata.getExtent(), updateCounter);
+ if (loggedErrorForTabletComparison) {
+log.info("AMCC Tablet {} files in memory are now same as in metadata
table {}",
+tabletMetadata.getExtent(), updateCounter);
+loggedErrorForTabletComparison = false;
+ } else {
+log.trace("AMCC Tablet {} files in memory are same as in metadata
table {}",
+tabletMetadata.getExtent(), updateCounter);
+ }
}
}
[accumulo] branch 2.1 updated: ensures sessions are always cleaned up (#3569)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new de02f025ce ensures sessions are always cleaned up (#3569)
de02f025ce is described below
commit de02f025ce5b25acfb86dbad9e9b96c7115d653f
Author: Keith Turner
AuthorDate: Mon Jul 10 11:27:46 2023 -0400
ensures sessions are always cleaned up (#3569)
This is a potential fix for #3512. It ensures that when a sesssions
cleanup method returns false that cleanup will be attempted again later.
---
.../accumulo/tserver/session/ScanSession.java | 4 ++
.../apache/accumulo/tserver/session/Session.java | 6 ++
.../accumulo/tserver/session/SessionManager.java | 84 +-
3 files changed, 62 insertions(+), 32 deletions(-)
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/ScanSession.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/ScanSession.java
index 3217fe1b8f..0fefcc1327 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/ScanSession.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/ScanSession.java
@@ -189,4 +189,8 @@ public abstract class ScanSession extends Session
implements ScanInfo {
return true;
}
+ @Override
+ public String toString() {
+return super.toString() + " tableId:" + getTableId();
+ }
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/Session.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/Session.java
index b1c14ca6e8..6e49833729 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/Session.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/Session.java
@@ -49,4 +49,10 @@ public class Session {
public boolean cleanup() {
return true;
}
+
+ @Override
+ public String toString() {
+return getClass().getSimpleName() + " " + state + " startTime:" +
startTime + " lastAccessTime:"
++ lastAccessTime + " client:" + client;
+ }
}
diff --git
a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java
b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java
index b8d605ebd8..f0f8a5de2a 100644
---
a/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java
+++
b/server/tserver/src/main/java/org/apache/accumulo/tserver/session/SessionManager.java
@@ -18,15 +18,22 @@
*/
package org.apache.accumulo.tserver.session;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static java.util.concurrent.TimeUnit.MINUTES;
+import static java.util.concurrent.TimeUnit.SECONDS;
+
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ScheduledFuture;
@@ -42,6 +49,7 @@ import
org.apache.accumulo.core.tabletserver.thrift.ActiveScan;
import org.apache.accumulo.core.tabletserver.thrift.ScanState;
import org.apache.accumulo.core.tabletserver.thrift.ScanType;
import org.apache.accumulo.core.util.MapCounter;
+import org.apache.accumulo.core.util.Retry;
import org.apache.accumulo.core.util.threads.ThreadPools;
import org.apache.accumulo.server.ServerContext;
import org.apache.accumulo.tserver.scan.ScanRunState;
@@ -61,7 +69,7 @@ public class SessionManager {
private final ConcurrentMap sessions = new
ConcurrentHashMap<>();
private final long maxIdle;
private final long maxUpdateIdle;
- private final List idleSessions = new ArrayList<>();
+ private final BlockingQueue deferredCleanupQueue = new
ArrayBlockingQueue<>(5000);
private final Long expiredSessionMarker = (long) -1;
private final AccumuloConfiguration aconf;
private final ServerContext ctx;
@@ -209,15 +217,39 @@ public class SessionManager {
}
if (doCleanup) {
-session.cleanup();
+cleanup(session);
}
}
return session;
}
+ private void cleanup(Session session) {
+if (!session.cleanup()) {
+ var retry = Retry.builder().infiniteRetries().retryAfter(25,
MILLISECONDS)
+ .incrementBy(25, MILLISECONDS).maxWait(5, SECONDS).backOffFactor(1.5)
+ .logInterval(1, MINUTES).createRetry();
+
+ while (!deferredCleanupQueue.offer(session)) {
+if (session.cleanup()) {
+ break;
+}
+
+try {
[accumulo] branch 2.1 updated: avoids calling toString on Text for isEmpty (#3572)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 767d28d99a avoids calling toString on Text for isEmpty (#3572)
767d28d99a is described below
commit 767d28d99aee1d460d05f82065346c5451a68b0c
Author: Keith Turner
AuthorDate: Mon Jul 10 11:28:36 2023 -0400
avoids calling toString on Text for isEmpty (#3572)
In profiling of an Accumulo map reduce job it was noticed that a good
bit of time was spent calling toString on a Text object to check if it
was empty. This commit avoids the toString() call and instead checks the
text length.
---
.../org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java| 2 +-
.../org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java | 2 +-
.../org/apache/accumulo/hadoopImpl/mapred/AccumuloRecordWriter.java | 2 +-
.../org/apache/accumulo/hadoopImpl/mapreduce/AccumuloRecordWriter.java | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
index 10d517c361..6dbecaee6b 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/mapred/AccumuloOutputFormat.java
@@ -425,7 +425,7 @@ public class AccumuloOutputFormat implements
OutputFormat {
*/
@Override
public void write(Text table, Mutation mutation) throws IOException {
- if (table == null || table.toString().isEmpty()) {
+ if (table == null || table.getLength() == 0) {
table = this.defaultTableName;
}
diff --git
a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
index 3c2193b3d3..c39c1013c1 100644
---
a/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
+++
b/core/src/main/java/org/apache/accumulo/core/client/mapreduce/AccumuloOutputFormat.java
@@ -426,7 +426,7 @@ public class AccumuloOutputFormat extends
OutputFormat {
*/
@Override
public void write(Text table, Mutation mutation) throws IOException {
- if (table == null || table.toString().isEmpty()) {
+ if (table == null || table.getLength() == 0) {
table = this.defaultTableName;
}
diff --git
a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapred/AccumuloRecordWriter.java
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapred/AccumuloRecordWriter.java
index ac012a1cf0..a29caf9c7f 100644
---
a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapred/AccumuloRecordWriter.java
+++
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapred/AccumuloRecordWriter.java
@@ -89,7 +89,7 @@ public class AccumuloRecordWriter implements
RecordWriter {
*/
@Override
public void write(Text table, Mutation mutation) throws IOException {
-if (table == null || table.toString().isEmpty()) {
+if (table == null || table.getLength() == 0) {
table = this.defaultTableName;
}
diff --git
a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/AccumuloRecordWriter.java
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/AccumuloRecordWriter.java
index 1399d8d0b7..fb21102ff2 100644
---
a/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/AccumuloRecordWriter.java
+++
b/hadoop-mapreduce/src/main/java/org/apache/accumulo/hadoopImpl/mapreduce/AccumuloRecordWriter.java
@@ -90,7 +90,7 @@ public class AccumuloRecordWriter extends
RecordWriter {
*/
@Override
public void write(Text table, Mutation mutation) throws IOException {
-if (table == null || table.toString().isEmpty()) {
+if (table == null || table.getLength() == 0) {
table = this.defaultTableName;
}
[accumulo] branch elasticity updated: reduces metadata reads and caches hosting request in client tablet cache (#3438)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new a3d9e1283a reduces metadata reads and caches hosting request in client
tablet cache (#3438)
a3d9e1283a is described below
commit a3d9e1283ac5221af77f2f2b1302befeab4eabd1
Author: Keith Turner
AuthorDate: Tue May 30 09:11:16 2023 -0400
reduces metadata reads and caches hosting request in client tablet cache
(#3438)
The code to host a tablet was reading from the metdata table to gather a
tablets hosting goal. The hosting goal was already present in the cache.
Used the value from the cache instead of reading from the metadata
table again.
Also added the hosting request column to the tablet cache and removed a
specialized caffine cache that was related to this. Moving it into the
tablet cache will help avoid request when other client processes have
already made the request, the specialized caffine cache was not enabling
this case.
fixes #3304
---
.../core/clientImpl/ClientTabletCache.java | 26 ---
.../core/clientImpl/ClientTabletCacheImpl.java | 80 ++
.../core/clientImpl/RootClientTabletCache.java | 7 +-
.../metadata/MetadataCachedTabletObtainer.java | 13 ++--
.../core/clientImpl/ClientTabletCacheImplTest.java | 28
.../test/functional/BulkSplitOptimizationIT.java | 2 +-
6 files changed, 82 insertions(+), 74 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java
index 71e0510978..fb8c4f7a8c 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/ClientTabletCache.java
@@ -276,34 +276,39 @@ public abstract class ClientTabletCache {
private final String tserverLocation;
private final String tserverSession;
private final TabletHostingGoal goal;
+private final boolean hostingRequested;
+
private final Long creationTime = System.nanoTime();
public CachedTablet(KeyExtent tablet_extent, String tablet_location,
String session,
-TabletHostingGoal goal) {
+TabletHostingGoal goal, boolean hostingRequested) {
checkArgument(tablet_extent != null, "tablet_extent is null");
checkArgument(tablet_location != null, "tablet_location is null");
checkArgument(session != null, "session is null");
this.tablet_extent = tablet_extent;
this.tserverLocation = interner.intern(tablet_location);
this.tserverSession = interner.intern(session);
- this.goal = goal;
+ this.goal = Objects.requireNonNull(goal);
+ this.hostingRequested = hostingRequested;
}
public CachedTablet(KeyExtent tablet_extent, Optional
tablet_location,
-Optional session, TabletHostingGoal goal) {
+Optional session, TabletHostingGoal goal, boolean
hostingRequested) {
checkArgument(tablet_extent != null, "tablet_extent is null");
this.tablet_extent = tablet_extent;
this.tserverLocation =
tablet_location.map(interner::intern).orElse(null);
this.tserverSession = session.map(interner::intern).orElse(null);
- this.goal = goal;
+ this.goal = Objects.requireNonNull(goal);
+ this.hostingRequested = hostingRequested;
}
-public CachedTablet(KeyExtent tablet_extent, TabletHostingGoal goal) {
+public CachedTablet(KeyExtent tablet_extent, TabletHostingGoal goal,
boolean hostingRequested) {
checkArgument(tablet_extent != null, "tablet_extent is null");
this.tablet_extent = tablet_extent;
this.tserverLocation = null;
this.tserverSession = null;
- this.goal = goal;
+ this.goal = Objects.requireNonNull(goal);
+ this.hostingRequested = hostingRequested;
}
@Override
@@ -312,14 +317,15 @@ public abstract class ClientTabletCache {
CachedTablet otl = (CachedTablet) o;
return getExtent().equals(otl.getExtent())
&& getTserverLocation().equals(otl.getTserverLocation())
-&& getTserverSession().equals(otl.getTserverSession()) &&
getGoal() == otl.getGoal();
+&& getTserverSession().equals(otl.getTserverSession()) &&
getGoal() == otl.getGoal()
+&& hostingRequested == otl.hostingRequested;
}
return false;
}
@Override
public int hashCode() {
- return Objects.hash(getExtent(), tserverLocation, tserverSession, goal);
+ return Objects.hash(getExtent(), tserverLocation, tserverSession, goal,
hostingRequested);
}
@Override
@@ -353,6
[accumulo] branch elasticity updated: adds validation to user split and new tests (#3431)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new 5608a9291a adds validation to user split and new tests (#3431)
5608a9291a is described below
commit 5608a9291a38b449c5dd7b1084a84719cf7c0294
Author: Keith Turner
AuthorDate: Fri May 26 19:09:08 2023 -0400
adds validation to user split and new tests (#3431)
---
.../accumulo/manager/FateServiceHandler.java | 31 +
.../apache/accumulo/test/functional/SplitIT.java | 39 +-
2 files changed, 69 insertions(+), 1 deletion(-)
diff --git
a/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
b/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
index 9cff4251dc..7b13303d70 100644
---
a/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
+++
b/server/manager/src/main/java/org/apache/accumulo/manager/FateServiceHandler.java
@@ -43,6 +43,7 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
+import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.apache.accumulo.core.client.AccumuloSecurityException;
@@ -75,6 +76,7 @@ import
org.apache.accumulo.core.manager.thrift.ThriftPropertyException;
import org.apache.accumulo.core.securityImpl.thrift.TCredentials;
import org.apache.accumulo.core.util.ByteBufferUtil;
import org.apache.accumulo.core.util.FastFormat;
+import org.apache.accumulo.core.util.TextUtil;
import org.apache.accumulo.core.util.Validator;
import org.apache.accumulo.core.util.tables.TableNameUtil;
import org.apache.accumulo.core.volume.Volume;
@@ -740,6 +742,35 @@ class FateServiceHandler implements FateService.Iface {
.map(ByteBufferUtil::toText).collect(Collectors.toCollection(TreeSet::new));
KeyExtent extent = new KeyExtent(tableId, endRow, prevEndRow);
+
+Predicate outOfBoundsTest =
+split -> !extent.contains(split) || split.equals(extent.endRow());
+
+if (splits.stream().anyMatch(outOfBoundsTest)) {
+ splits.stream().filter(outOfBoundsTest).forEach(split -> log
+ .warn("split for {} is out of bounds : {}", extent,
TextUtil.truncate(split)));
+
+ throw new ThriftTableOperationException(tableId.canonical(), null,
tableOp,
+ TableOperationExceptionType.OTHER,
+ "Split is outside bounds of tablet or equal to the tablets
endrow, see warning in logs for more information.");
+}
+
+var maxSplitSize = manager.getContext().getTableConfiguration(tableId)
+.getAsBytes(Property.TABLE_MAX_END_ROW_SIZE);
+
+Predicate oversizedTest = split -> split.getLength() >
maxSplitSize;
+
+if (splits.stream().anyMatch(oversizedTest)) {
+ splits.stream().filter(oversizedTest)
+ .forEach(split -> log.warn(
+ "split exceeds max configured split size len:{} max:{}
extent:{} split:{}",
+ split.getLength(), maxSplitSize, extent,
TextUtil.truncate(split)));
+
+ throw new ThriftTableOperationException(tableId.canonical(), null,
tableOp,
+ TableOperationExceptionType.OTHER,
+ "Length of requested split exceeds tables configured max, see
warning in logs for more information.");
+}
+
manager.requestUnassignment(extent, opid);
goalMessage = "Splitting " + extent + " for user into " +
(splits.size() + 1) + " tablets";
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java
index f143c91480..3a342b78e8 100644
--- a/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/functional/SplitIT.java
@@ -20,16 +20,20 @@ package org.apache.accumulo.test.functional;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assumptions.assumeTrue;
import java.time.Duration;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.TreeSet;
import org.apache.accumulo.core.client.Accumulo;
import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.AccumuloException;
import org.apache.accumulo.core.client.Scanner;
import org.apache.accumulo.core.client.admin.InstanceOperations;
import org.apache.accumulo.core.client.admi
[accumulo] branch elasticity updated (a3d9e1283a -> 1893fd4dad)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from a3d9e1283a reduces metadata reads and caches hosting request in client tablet cache (#3438) add 1893fd4dad moves hosting request processing from tserver to manager (#3437) No new revisions were added by this update. Summary of changes: .../core/clientImpl/ClientTabletCacheImpl.java |2 +- .../accumulo/core/metadata/schema/Ample.java |5 + .../core/manager/thrift/ManagerClientService.java | 1435 .../thrift/TabletManagementClientService.java | 1319 -- core/src/main/thrift/manager.thrift|9 + core/src/main/thrift/tabletmgmt.thrift | 10 - .../metadata/ConditionalTabletMutatorImpl.java | 12 + .../manager/ManagerClientServiceHandler.java | 61 + .../apache/accumulo/manager/split/Splitter.java|2 +- .../accumulo/tserver/TabletClientHandler.java | 33 - .../accumulo/test/performance/NullTserver.java |4 - 11 files changed, 1524 insertions(+), 1368 deletions(-)
[accumulo] branch 2.1 updated: adds tests for file misnormalization (#3435)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new ee8da3cc27 adds tests for file misnormalization (#3435)
ee8da3cc27 is described below
commit ee8da3cc2702b1d2b335c2275a994c535012f029
Author: Keith Turner
AuthorDate: Fri Jun 2 15:05:55 2023 -0400
adds tests for file misnormalization (#3435)
---
.../test/functional/FileNormalizationIT.java | 238 +
1 file changed, 238 insertions(+)
diff --git
a/test/src/main/java/org/apache/accumulo/test/functional/FileNormalizationIT.java
b/test/src/main/java/org/apache/accumulo/test/functional/FileNormalizationIT.java
new file mode 100644
index 00..25a54c97cb
--- /dev/null
+++
b/test/src/main/java/org/apache/accumulo/test/functional/FileNormalizationIT.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.accumulo.test.functional;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.accumulo.core.client.Accumulo;
+import org.apache.accumulo.core.client.AccumuloClient;
+import org.apache.accumulo.core.client.MutationsRejectedException;
+import org.apache.accumulo.core.client.Scanner;
+import org.apache.accumulo.core.client.admin.CompactionConfig;
+import org.apache.accumulo.core.client.admin.NewTableConfiguration;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.Mutation;
+import org.apache.accumulo.core.data.TableId;
+import org.apache.accumulo.core.dataImpl.KeyExtent;
+import org.apache.accumulo.core.metadata.MetadataTable;
+import
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.DataFileColumnFamily;
+import org.apache.accumulo.core.security.TablePermission;
+import org.apache.accumulo.harness.SharedMiniClusterBase;
+import org.apache.accumulo.test.TestIngest;
+import org.apache.accumulo.test.VerifyIngest;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * These test check that Accumulo handles misnormalized files in the metadata
table correctly. If
+ * Accumulo code reads a misnormalized file from the metadata table,
normalizes it, and then tries
+ * to update the metadata table then the key will not match. The mismatch
could result in duplicate
+ * entries.
+ */
+public class FileNormalizationIT extends SharedMiniClusterBase {
+
+ private static final Logger log =
LoggerFactory.getLogger(FileNormalizationIT.class);
+
+ @BeforeAll
+ public static void setup() throws Exception {
+SharedMiniClusterBase.startMiniCluster();
+ }
+
+ @AfterAll
+ public static void teardown() {
+SharedMiniClusterBase.stopMiniCluster();
+ }
+
+ @Test
+ public void testSplits() throws Exception {
+try (AccumuloClient client =
Accumulo.newClient().from(getClientProps()).build()) {
+ String table = getUniqueNames(1)[0];
+
+ client.tableOperations().create(table);
+ VerifyIngest.VerifyParams params =
+ new VerifyIngest.VerifyParams(getClientProps(), table, 100_000);
+ TestIngest.ingest(client, params);
+
+ client.tableOperations().flush(table, null, null, true);
+
+ misnormalizeFiles(client, table);
+
+ var splits = TestIngest.getSplitPoints(params.startRow, params.startRow
+ params.rows, 2);
+ assertEquals(1, splits.size());
+
+ client.tableOperations().addSplits(table, splits);
+
+ HashSet paths = new HashSet<>();
+
+ VerifyIngest.verifyIngest(client, params);
+
+ try (var scanner = createMetadataFileScanner(client, table)) {
+scanner.forEach((k, v) -> {
+ var row = k.getRowData().toString();
+ var qual = k.getC
[accumulo] branch main updated (256e277052 -> 775cd62fa3)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from 256e277052 Merge branch '2.1' add ee8da3cc27 adds tests for file misnormalization (#3435) new 775cd62fa3 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../test/functional/FileNormalizationIT.java | 238 + 1 file changed, 238 insertions(+) create mode 100644 test/src/main/java/org/apache/accumulo/test/functional/FileNormalizationIT.java
[accumulo] 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 775cd62fa381796ad5a76ad716313cfdd4216be5 Merge: 256e277052 ee8da3cc27 Author: Keith Turner AuthorDate: Fri Jun 2 15:07:51 2023 -0400 Merge branch '2.1' .../test/functional/FileNormalizationIT.java | 238 + 1 file changed, 238 insertions(+)
[accumulo] branch elasticity updated: executes user initiated splits in manager (#3425)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/elasticity by this push:
new c08139caf5 executes user initiated splits in manager (#3425)
c08139caf5 is described below
commit c08139caf511dee6b7166c781a6169aa420dbdc4
Author: Keith Turner
AuthorDate: Thu May 25 14:27:29 2023 -0400
executes user initiated splits in manager (#3425)
Modifies the user API for adding splits to execute a fate operation
instead of calling the tablet server. Now user initiated splits can
happen without having having to host a tablet.
---
.../core/clientImpl/TableOperationsImpl.java | 274 -
.../core/manager/state/TabletManagement.java | 9 +-
.../core/metadata/schema/TabletMetadata.java | 7 +-
.../core/clientImpl/thrift/TableOperation.java | 5 +-
.../core/manager/thrift/FateOperation.java | 5 +-
core/src/main/thrift/client.thrift | 1 +
core/src/main/thrift/manager.thrift| 1 +
.../manager/state/TabletManagementIterator.java| 17 +-
.../accumulo/manager/FateServiceHandler.java | 53
.../java/org/apache/accumulo/manager/Manager.java | 4 +
.../manager/tableOps/split/DeleteOperationIds.java | 6 +
.../accumulo/manager/tableOps/split/PreSplit.java | 2 +-
.../test/functional/ManagerAssignmentIT.java | 61 -
.../functional/TabletManagementIteratorIT.java | 28 ++-
14 files changed, 285 insertions(+), 188 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
index 37dbe8488a..a063cff241 100644
---
a/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
+++
b/core/src/main/java/org/apache/accumulo/core/clientImpl/TableOperationsImpl.java
@@ -59,10 +59,11 @@ import java.util.Set;
import java.util.SortedSet;
import java.util.TreeMap;
import java.util.TreeSet;
-import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.function.Predicate;
@@ -140,8 +141,6 @@ import
org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl;
import org.apache.accumulo.core.security.Authorizations;
import org.apache.accumulo.core.summary.SummarizerConfigurationUtil;
import org.apache.accumulo.core.summary.SummaryCollection;
-import org.apache.accumulo.core.tablet.thrift.TabletManagementClientService;
-import org.apache.accumulo.core.tabletserver.thrift.NotServingTabletException;
import org.apache.accumulo.core.trace.TraceUtil;
import org.apache.accumulo.core.util.LocalityGroupUtil;
import
org.apache.accumulo.core.util.LocalityGroupUtil.LocalityGroupConfigurationError;
@@ -155,7 +154,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
-import org.apache.thrift.TApplicationException;
import org.apache.thrift.TException;
import org.apache.thrift.TSerializer;
import org.apache.thrift.transport.TTransportException;
@@ -164,7 +162,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
-import com.google.common.net.HostAndPort;
public class TableOperationsImpl extends TableOperationsHelper {
@@ -435,198 +432,143 @@ public class TableOperationsImpl extends
TableOperationsHelper {
}
}
- private static class SplitEnv {
-private final String tableName;
-private final TableId tableId;
-private final ExecutorService executor;
-private final CountDownLatch latch;
-private final AtomicReference exception;
-
-SplitEnv(String tableName, TableId tableId, ExecutorService executor,
CountDownLatch latch,
-AtomicReference exception) {
- this.tableName = tableName;
- this.tableId = tableId;
- this.executor = executor;
- this.latch = latch;
- this.exception = exception;
-}
- }
-
- private class SplitTask implements Runnable {
-
-private List splits;
-private SplitEnv env;
-
-SplitTask(SplitEnv env, List splits) {
- this.env = env;
- this.splits = splits;
-}
+ /**
+ * On the server side the fate operation will exit w/o an error if the
tablet requested to split
+ * does not exist. When this happens it will also return an empty string. In
the case where the
+ * fate operation successfully splits the tablet it will return
[accumulo] branch elasticity updated (4ba6c499e2 -> 83cfa8fc28)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a change to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
from 4ba6c499e2 Merge branch 'main' into elasticity
add 30e30bda56 fail when unknown config seen in cluster.yaml (#3452)
add b202483d62 Merge remote-tracking branch 'upstream/2.1'
add 83cfa8fc28 Merge branch 'main' into elasticity
No new revisions were added by this update.
Summary of changes:
.../accumulo/core/conf/cluster/ClusterConfigParser.java | 17 +
.../core/conf/cluster/ClusterConfigParserTest.java | 17 +
.../conf/cluster/{cluster.yaml => bad-cluster.yaml} | 7 +++
3 files changed, 41 insertions(+)
copy
core/src/test/resources/org/apache/accumulo/core/conf/cluster/{cluster.yaml =>
bad-cluster.yaml} (89%)
[accumulo] branch 2.1 updated: fail when unknown config seen in cluster.yaml (#3452)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 30e30bda56 fail when unknown config seen in cluster.yaml (#3452)
30e30bda56 is described below
commit 30e30bda565c70880be369a386d42b380603acce
Author: Keith Turner
AuthorDate: Tue Jun 6 22:40:01 2023 -0400
fail when unknown config seen in cluster.yaml (#3452)
Parsing of the cluster.yaml file would ignore unrecognized config keys
before this changes. After this change it should throw an exception.
Co-authored-by: Christopher Tubbs
---
.../core/conf/cluster/ClusterConfigParser.java | 17 +++
.../core/conf/cluster/ClusterConfigParserTest.java | 17 +++
.../accumulo/core/conf/cluster/bad-cluster.yaml| 54 ++
3 files changed, 88 insertions(+)
diff --git
a/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
b/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
index de43584de0..4f41cbef3e 100644
---
a/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
+++
b/core/src/main/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParser.java
@@ -29,6 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.yaml.snakeyaml.Yaml;
@@ -40,6 +41,16 @@ public class ClusterConfigParser {
private static final String PROPERTY_FORMAT = "%s=\"%s\"%n";
private static final String[] SECTIONS = new String[] {"manager", "monitor",
"gc", "tserver"};
+ private static final Set VALID_CONFIG_KEYS = Set.of("manager",
"monitor", "gc", "tserver",
+ "tservers_per_host", "sservers_per_host", "compaction.coordinator");
+
+ private static final Set VALID_CONFIG_PREFIXES =
+ Set.of("compaction.compactor.", "sserver.");
+
+ private static final Predicate VALID_CONFIG_SECTIONS =
+ section -> VALID_CONFIG_KEYS.contains(section)
+ || VALID_CONFIG_PREFIXES.stream().anyMatch(section::startsWith);
+
@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "paths not
set by user input")
public static Map parseConfiguration(String configFile)
throws IOException {
Map results = new HashMap<>();
@@ -87,6 +98,12 @@ public class ClusterConfigParser {
public static void outputShellVariables(Map config,
PrintStream out) {
+// find invalid config sections and point the user to the first one
+config.keySet().stream().filter(VALID_CONFIG_SECTIONS.negate()).findFirst()
+.ifPresent(section -> {
+ throw new IllegalArgumentException("Unknown configuration section :
" + section);
+});
+
for (String section : SECTIONS) {
if (config.containsKey(section)) {
out.printf(PROPERTY_FORMAT, section.toUpperCase() + "_HOSTS",
config.get(section));
diff --git
a/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
b/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
index 82ca8a41d0..ef2c2382bc 100644
---
a/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/conf/cluster/ClusterConfigParserTest.java
@@ -21,10 +21,12 @@ package org.apache.accumulo.core.conf.cluster;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
@@ -242,4 +244,19 @@ public class ClusterConfigParserTest {
assertEquals(expected, actual);
}
+ @Test
+ public void testFileWithUnknownSections() throws Exception {
+URL configFile = ClusterConfigParserTest.class
+
.getResource("/org/apache/accumulo/core/conf/cluster/bad-cluster.yaml");
+assertNotNull(configFile);
+
+Map contents =
+ClusterConfigParser.parseConfiguration(new
File(configFile.toURI()).getAbsolutePath());
+
+try (var baos = new ByteArrayOutputStream(); var ps = new
PrintStream(baos)) {
+ var exception = assertThrows(IllegalArgumentException.class,
+ () -> ClusterConfigParser.outputShellVariables(contents,
