(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch elasticity
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 861d0fdfc2b7bdd6da72e0e5652c027643ec5a49
Merge: 1c706c541e 1ae2b17e2b
Author: Christopher L. Shannon
AuthorDate: Sat Apr 6 16:11:28 2024 -0400
Merge branch 'main' into elasticity
.../core/metadata/schema/TabletMetadata.java | 389 ++---
.../core/metadata/schema/TabletMetadataTest.java | 105 +-
2 files changed, 362 insertions(+), 132 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index be7d2fb056,f2ad719fd8..6f2ffd6237
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@@ -92,44 -82,63 +92,80 @@@ import com.google.common.collect.Immuta
import com.google.common.net.HostAndPort;
public class TabletMetadata {
+
private static final Logger log =
LoggerFactory.getLogger(TabletMetadata.class);
- private TableId tableId;
- private Text prevEndRow;
- private boolean sawPrevEndRow = false;
- private Text endRow;
- private Location location;
- private Map files;
- private List scans;
- private Map loadedFiles;
- private SelectedFiles selectedFiles;
- private EnumSet fetchedCols;
- private KeyExtent extent;
- private Location last;
- private SuspendingTServer suspend;
- private String dirName;
- private MetadataTime time;
- private String cloned;
- private SortedMap keyValues;
- private OptionalLong flush = OptionalLong.empty();
- private OptionalLong flushNonce = OptionalLong.empty();
- private List logs;
- private Map extCompactions;
- private boolean merged;
- private TabletAvailability availability = TabletAvailability.ONDEMAND;
- private boolean onDemandHostingRequested = false;
- private TabletOperationId operationId;
- private boolean futureAndCurrentLocationSet = false;
- private Set compacted;
- private Set userCompactionsRequested;
- private UnSplittableMetadata unSplittableMetadata;
- private Supplier fileSize;
+ private final TableId tableId;
+ private final Text prevEndRow;
+ private final boolean sawPrevEndRow;
- private final Text oldPrevEndRow;
- private final boolean sawOldPrevEndRow;
+ private final Text endRow;
+ private final Location location;
+ private final Map files;
+ private final List scans;
- private final Map loadedFiles;
++ private final Map loadedFiles;
++ private final SelectedFiles selectedFiles;
+ private final EnumSet fetchedCols;
+ private final Supplier extent;
+ private final Location last;
+ private final SuspendingTServer suspend;
+ private final String dirName;
+ private final MetadataTime time;
+ private final String cloned;
+ private final SortedMap keyValues;
+ private final OptionalLong flush;
++ private final OptionalLong flushNonce;
+ private final List logs;
- private final OptionalLong compact;
- private final Double splitRatio;
- private final Map
extCompactions;
++ private final Map extCompactions;
+ private final boolean merged;
++ private final TabletAvailability availability;
++ private final boolean onDemandHostingRequested;
++ private final TabletOperationId operationId;
++ private final boolean futureAndCurrentLocationSet;
++ private final Set compacted;
++ private final Set userCompactionsRequested;
++ private final UnSplittableMetadata unSplittableMetadata;
++ private final Supplier fileSize;
+
+ private TabletMetadata(Builder tmBuilder) {
+ this.tableId = tmBuilder.tableId;
+ this.prevEndRow = tmBuilder.prevEndRow;
+ this.sawPrevEndRow = tmBuilder.sawPrevEndRow;
-this.oldPrevEndRow = tmBuilder.oldPrevEndRow;
-this.sawOldPrevEndRow = tmBuilder.sawOldPrevEndRow;
+ this.endRow = tmBuilder.endRow;
+ this.location = tmBuilder.location;
+ this.files = Objects.requireNonNull(tmBuilder.files.build());
+ this.scans = Objects.requireNonNull(tmBuilder.scans.build());
+ this.loadedFiles = tmBuilder.loadedFiles.build();
++this.selectedFiles = tmBuilder.selectedFiles;
+ this.fetchedCols = Objects.requireNonNull(tmBuilder.fetchedCols);
+ this.last = tmBuilder.last;
+ this.suspend = tmBuilder.suspend;
+ this.dirName = tmBuilder.dirName;
+ this.time = tmBuilder.time;
+ this.cloned = tmBuilder.cloned;
+ this.keyValues =
Optional.ofNullable(tmBuilder.keyValues).map(ImmutableSortedMap.Builder::build)
+ .orElse(null);
+ this.flush = tmBuilder.flush;
++this.flushNonce = tmBuilder.flushNonce;
+ this.logs = Objects.requireNonNull(tmBuilder.logs.build());
-this.compact = Objects.requireNonNull(tmBuilder.compact);
-this.splitRatio = tmBuilder.splitRatio;
+ this.extCompactions =
Objects.requireNonNull(tmBuilder.extCompactions.build());
+
(accumulo) branch elasticity updated (1c706c541e -> 861d0fdfc2)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 1c706c541e Merge branch 'main' into elasticity add 1ae2b17e2b Update TabletMetadata to be immutable (#4437) new 861d0fdfc2 Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../core/metadata/schema/TabletMetadata.java | 389 ++--- .../core/metadata/schema/TabletMetadataTest.java | 105 +- 2 files changed, 362 insertions(+), 132 deletions(-)
(accumulo) branch main updated: Update TabletMetadata to be immutable (#4437)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/main by this push:
new 1ae2b17e2b Update TabletMetadata to be immutable (#4437)
1ae2b17e2b is described below
commit 1ae2b17e2bb94c922b5a3a7482cf92fb5ae23883
Author: Christopher L. Shannon
AuthorDate: Sat Apr 6 15:52:31 2024 -0400
Update TabletMetadata to be immutable (#4437)
This commit refactors all of the fields to be final for TabletMetadata.
The object is already treated as immutable in practice, but
previously didn't declare fields final so this could lead to potential
future bugs and also means not being able to guarantee the state of a
field which is now possible.
---
.../core/metadata/schema/TabletMetadata.java | 302 +++--
.../core/metadata/schema/TabletMetadataTest.java | 101 ++-
2 files changed, 313 insertions(+), 90 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index b98d9e8540..f2ad719fd8 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@ -74,38 +74,70 @@ import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.collect.ImmutableSortedMap.Builder;
import com.google.common.net.HostAndPort;
public class TabletMetadata {
private static final Logger log =
LoggerFactory.getLogger(TabletMetadata.class);
- private TableId tableId;
- private Text prevEndRow;
- private boolean sawPrevEndRow = false;
- private Text oldPrevEndRow;
- private boolean sawOldPrevEndRow = false;
- private Text endRow;
- private Location location;
- private Map files;
- private List scans;
- private Map loadedFiles;
- private EnumSet fetchedCols;
- private KeyExtent extent;
- private Location last;
- private SuspendingTServer suspend;
- private String dirName;
- private MetadataTime time;
- private String cloned;
- private SortedMap keyValues;
- private OptionalLong flush = OptionalLong.empty();
- private List logs;
- private OptionalLong compact = OptionalLong.empty();
- private Double splitRatio = null;
- private Map extCompactions;
- private boolean merged;
+ private final TableId tableId;
+ private final Text prevEndRow;
+ private final boolean sawPrevEndRow;
+ private final Text oldPrevEndRow;
+ private final boolean sawOldPrevEndRow;
+ private final Text endRow;
+ private final Location location;
+ private final Map files;
+ private final List scans;
+ private final Map loadedFiles;
+ private final EnumSet fetchedCols;
+ private final Supplier extent;
+ private final Location last;
+ private final SuspendingTServer suspend;
+ private final String dirName;
+ private final MetadataTime time;
+ private final String cloned;
+ private final SortedMap keyValues;
+ private final OptionalLong flush;
+ private final List logs;
+ private final OptionalLong compact;
+ private final Double splitRatio;
+ private final Map
extCompactions;
+ private final boolean merged;
+
+ private TabletMetadata(Builder tmBuilder) {
+this.tableId = tmBuilder.tableId;
+this.prevEndRow = tmBuilder.prevEndRow;
+this.sawPrevEndRow = tmBuilder.sawPrevEndRow;
+this.oldPrevEndRow = tmBuilder.oldPrevEndRow;
+this.sawOldPrevEndRow = tmBuilder.sawOldPrevEndRow;
+this.endRow = tmBuilder.endRow;
+this.location = tmBuilder.location;
+this.files = Objects.requireNonNull(tmBuilder.files.build());
+this.scans = Objects.requireNonNull(tmBuilder.scans.build());
+this.loadedFiles = tmBuilder.loadedFiles.build();
+this.fetchedCols = Objects.requireNonNull(tmBuilder.fetchedCols);
+this.last = tmBuilder.last;
+this.suspend = tmBuilder.suspend;
+this.dirName = tmBuilder.dirName;
+this.time = tmBuilder.time;
+this.cloned = tmBuilder.cloned;
+this.keyValues =
Optional.ofNullable(tmBuilder.keyValues).map(ImmutableSortedMap.Builder::build)
+.orElse(null);
+this.flush = tmBuilder.flush;
+this.logs = Objects.requireNonNull(tmBuilder.logs.build());
+this.compact = Objects.requireNonNull(tmBuilder.compact);
+this.splitRatio = tmBuilder.splitRatio;
+this.extCompactions =
Objects.requireNonNull(tmBuilder.extCompactions.build());
+this.merged = tmBuilder.merged;
+this.extent =
+Suppliers.memoize(() -> new
(accumulo) branch elasticity updated (77e95f95b9 -> 1c706c541e)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git from 77e95f95b9 Existing logging in ManagerClientServiceHandler is sufficient, removed todo (#4433) add 90bcb465fe Ensure getTabletState() in TabletMetadata does not allow a null extent (#4438) add 84e8557ab8 Merge branch '2.1' new 1c706c541e Merge branch 'main' into elasticity The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
(accumulo) 01/01: Merge branch 'main' into elasticity
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git commit 1c706c541e1e81fa2a8c87b72e3c1a59d52a282a Merge: 77e95f95b9 84e8557ab8 Author: Christopher L. Shannon AuthorDate: Sat Apr 6 15:34:09 2024 -0400 Merge branch 'main' into elasticity
(accumulo) branch main updated (b912506d43 -> 84e8557ab8)
This is an automated email from the ASF dual-hosted git repository. cshannon pushed a change to branch main in repository https://gitbox.apache.org/repos/asf/accumulo.git from b912506d43 avoids acquiring recovery lock when tablet has no wals (#4429) add 90bcb465fe Ensure getTabletState() in TabletMetadata does not allow a null extent (#4438) new 84e8557ab8 Merge branch '2.1' The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../org/apache/accumulo/core/metadata/TabletLocationState.java| 3 ++- .../org/apache/accumulo/core/metadata/schema/TabletMetadata.java | 4 +++- .../apache/accumulo/core/metadata/schema/TabletMetadataTest.java | 8 3 files changed, 13 insertions(+), 2 deletions(-)
(accumulo) 01/01: Merge branch '2.1'
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/accumulo.git
commit 84e8557ab8ddabd23c8813d9dbed2e1223d99646
Merge: b912506d43 90bcb465fe
Author: Christopher L. Shannon
AuthorDate: Sat Apr 6 15:23:23 2024 -0400
Merge branch '2.1'
.../org/apache/accumulo/core/metadata/TabletLocationState.java| 3 ++-
.../org/apache/accumulo/core/metadata/schema/TabletMetadata.java | 4 +++-
.../apache/accumulo/core/metadata/schema/TabletMetadataTest.java | 8
3 files changed, 13 insertions(+), 2 deletions(-)
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/TabletLocationState.java
index eeaa5cde95,0a70594027..4c2421b4e8
---
a/core/src/main/java/org/apache/accumulo/core/metadata/TabletLocationState.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/TabletLocationState.java
@@@ -57,8 -57,9 +58,8 @@@ public class TabletLocationState
}
public TabletLocationState(KeyExtent extent, Location future, Location
current, Location last,
- SuspendingTServer suspend, Collection> walogs,
boolean chopped)
- throws BadLocationStateException {
+ SuspendingTServer suspend, Collection walogs) throws
BadLocationStateException {
- this.extent = extent;
+ this.extent = Objects.requireNonNull(extent);
this.future = validateLocation(future,
TabletMetadata.LocationType.FUTURE);
this.current = validateLocation(current,
TabletMetadata.LocationType.CURRENT);
this.last = validateLocation(last, TabletMetadata.LocationType.LAST);
diff --cc
core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index af31605d51,c1e816109a..b98d9e8540
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@@ -371,7 -373,9 +371,9 @@@ public class TabletMetadata
future = location;
}
// only care about the state so don't need walogs and chopped params
- var tls = new TabletLocationState(extent, future, current, last,
suspend, null);
+ // Use getExtent() when passing the extent as the private reference may
not have been
+ // initialized yet. This will also ensure PREV_ROW was fetched
- var tls = new TabletLocationState(getExtent(), future, current, last,
suspend, null, false);
++ var tls = new TabletLocationState(getExtent(), future, current, last,
suspend, null);
return tls.getState(liveTServers);
} catch (TabletLocationState.BadLocationStateException blse) {
throw new IllegalArgumentException("Error creating
TabletLocationState", blse);
(accumulo) branch 2.1 updated: Ensure getTabletState() in TabletMetadata does not allow a null extent (#4438)
This is an automated email from the ASF dual-hosted git repository.
cshannon pushed a commit to branch 2.1
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/2.1 by this push:
new 90bcb465fe Ensure getTabletState() in TabletMetadata does not allow a
null extent (#4438)
90bcb465fe is described below
commit 90bcb465fe672ec0181bbcd8acc1d6f4f22e66be
Author: Christopher L. Shannon
AuthorDate: Sat Apr 6 15:20:07 2024 -0400
Ensure getTabletState() in TabletMetadata does not allow a null extent
(#4438)
Previously the private reference for the extent inside TabletMetadata
was passed to the TabletLocationState constructor inside of
getTabletState(). The extent may not have been loaded at this point as
it is lazy loaded by the getExtent() method and requires PREV_ROW to
have been fetched. This change now uses the getter to make sure we do
not inadvertently pass a null extent which is invalid.
---
.../org/apache/accumulo/core/metadata/TabletLocationState.java| 3 ++-
.../org/apache/accumulo/core/metadata/schema/TabletMetadata.java | 4 +++-
.../apache/accumulo/core/metadata/schema/TabletMetadataTest.java | 8
3 files changed, 13 insertions(+), 2 deletions(-)
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/TabletLocationState.java
b/core/src/main/java/org/apache/accumulo/core/metadata/TabletLocationState.java
index d0cd66300b..0a70594027 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/TabletLocationState.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/TabletLocationState.java
@@ -22,6 +22,7 @@ import static java.util.Objects.requireNonNull;
import java.util.Collection;
import java.util.Collections;
+import java.util.Objects;
import java.util.Set;
import org.apache.accumulo.core.dataImpl.KeyExtent;
@@ -58,7 +59,7 @@ public class TabletLocationState {
public TabletLocationState(KeyExtent extent, Location future, Location
current, Location last,
SuspendingTServer suspend, Collection> walogs,
boolean chopped)
throws BadLocationStateException {
-this.extent = extent;
+this.extent = Objects.requireNonNull(extent);
this.future = validateLocation(future, TabletMetadata.LocationType.FUTURE);
this.current = validateLocation(current,
TabletMetadata.LocationType.CURRENT);
this.last = validateLocation(last, TabletMetadata.LocationType.LAST);
diff --git
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
index 323f0cb142..c1e816109a 100644
---
a/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
+++
b/core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadata.java
@@ -373,7 +373,9 @@ public class TabletMetadata {
future = location;
}
// only care about the state so don't need walogs and chopped params
- var tls = new TabletLocationState(extent, future, current, last,
suspend, null, false);
+ // Use getExtent() when passing the extent as the private reference may
not have been
+ // initialized yet. This will also ensure PREV_ROW was fetched
+ var tls = new TabletLocationState(getExtent(), future, current, last,
suspend, null, false);
return tls.getState(liveTServers);
} catch (TabletLocationState.BadLocationStateException blse) {
throw new IllegalArgumentException("Error creating TabletLocationState",
blse);
diff --git
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
index 67a22ccd3f..7d9284ba00 100644
---
a/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
+++
b/core/src/test/java/org/apache/accumulo/core/metadata/schema/TabletMetadataTest.java
@@ -26,6 +26,7 @@ import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSec
import static
org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.TIME_COLUMN;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LAST;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.LOCATION;
+import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.PREV_ROW;
import static
org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType.SUSPEND;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -196,6 +197,13 @@ public class TabletMetadataTest {
.put(ser1.getHostPort());
SortedMap rowMap = toRowMap(mutation);
+// PREV_ROW was not fetched
+final var missingPrevRow =
+TabletMetadata.convertRow(rowMap.entrySet().iterator(), colsToFetch,
false);
