(accumulo) branch elasticity updated: Fixed RecoveryIT timeout issue (#4271)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/elasticity by this push: new da7b3b005e Fixed RecoveryIT timeout issue (#4271) da7b3b005e is described below commit da7b3b005e13276c202b4f337496f9ecf2c039e5 Author: Dave Marion AuthorDate: Thu Feb 15 16:33:07 2024 -0500 Fixed RecoveryIT timeout issue (#4271) Test was waiting for 60s to confirm that recovery logs were sorted. However, the default interval for checking if recovery logs needed to be sorted is 1m. Increased the timeout in the test so that it would not fail. --- test/src/main/java/org/apache/accumulo/test/RecoveryIT.java | 6 +- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/src/main/java/org/apache/accumulo/test/RecoveryIT.java b/test/src/main/java/org/apache/accumulo/test/RecoveryIT.java index c8fb440eea..6da4cd9d77 100644 --- a/test/src/main/java/org/apache/accumulo/test/RecoveryIT.java +++ b/test/src/main/java/org/apache/accumulo/test/RecoveryIT.java @@ -18,6 +18,7 @@ */ package org.apache.accumulo.test; +import static org.apache.accumulo.core.util.compaction.ExternalCompactionUtil.getCompactorAddrs; import static org.apache.accumulo.harness.AccumuloITBase.MINI_CLUSTER_ONLY; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; @@ -142,7 +143,10 @@ public class RecoveryIT extends AccumuloClusterHarness { // Stop any running Compactors and ScanServers control.stopAllServers(ServerType.COMPACTOR); + Wait.waitFor(() -> getCompactorAddrs(getCluster().getServerContext()).size() == 0, 60_000); + control.stopAllServers(ServerType.SCAN_SERVER); + Wait.waitFor(() -> ((ClientContext) c).getScanServers().size() == 0, 60_000); // Kill the TabletServer in resource group that is hosting the table List procs = control.getTabletServers(RESOURCE_GROUP); @@ -181,7 +185,7 @@ public class RecoveryIT extends AccumuloClusterHarness { } // Confirm sorting completed - Wait.waitFor(() -> logSortingCompleted(c, tid) == true, 60_000); + Wait.waitFor(() -> logSortingCompleted(c, tid) == true, 120_000); // Start the tablet servers so that the Manager // can assign the table and so that recovery can be completed.
(accumulo) branch elasticity updated: Fixed MemoryStarvedScanIT timeout (#4270)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/elasticity by this push: new 5cf9953142 Fixed MemoryStarvedScanIT timeout (#4270) 5cf9953142 is described below commit 5cf99531425ccde5199bc2de344a5a6cce26baa5 Author: Dave Marion AuthorDate: Thu Feb 15 16:22:35 2024 -0500 Fixed MemoryStarvedScanIT timeout (#4270) MemoryStarvedScanIT was timing out because the IT was deleting tables as part of the test and scans on the new FATE table were being blocked because of the low memory condition. Adjusted the check for whether the table is a user table or not to include all tables in the accumulo namespace. --- .../src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java index 28ad0e0744..ff67ed0469 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/TabletBase.java @@ -42,6 +42,7 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator; import org.apache.accumulo.core.iterators.YieldCallback; import org.apache.accumulo.core.iteratorsImpl.system.IterationInterruptedException; import org.apache.accumulo.core.iteratorsImpl.system.SourceSwitchingIterator; +import org.apache.accumulo.core.metadata.AccumuloTable; import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.sample.impl.SamplerConfigurationImpl; @@ -92,7 +93,7 @@ public abstract class TabletBase { this.context = server.getContext(); this.server = server; this.extent = extent; -this.isUserTable = !extent.isMeta(); +this.isUserTable = !AccumuloTable.allTableIds().contains(extent.tableId()); TableConfiguration tblConf = context.getTableConfiguration(extent.tableId()); if (tblConf == null) {
(accumulo) branch elasticity updated: Resolved timeout issue in MemoryStarvedMajCIT (#4267)
This is an automated email from the ASF dual-hosted git repository. dlmarion pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/elasticity by this push: new 335592c68d Resolved timeout issue in MemoryStarvedMajCIT (#4267) 335592c68d is described below commit 335592c68deb4fa59e00decc4d3fd72251f2d963 Author: Dave Marion AuthorDate: Thu Feb 15 14:33:06 2024 -0500 Resolved timeout issue in MemoryStarvedMajCIT (#4267) It appears that the test was stopping the Compactors, but not waiting for their ZooKeeper locks to be removed, before moving on and checking the number of zookeeper locks. I think what was happening is that the test code moved forward thinking that the Compactors were stopped, but the Compactors were actually stopping after this point and during the next step in the test. --- .../util/compaction/ExternalCompactionUtil.java | 2 +- .../miniclusterImpl/MiniAccumuloClusterControl.java | 4 .../test/functional/MemoryStarvedMajCIT.java| 21 ++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java index 697f079d40..0d42f659c8 100644 --- a/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java +++ b/core/src/main/java/org/apache/accumulo/core/util/compaction/ExternalCompactionUtil.java @@ -114,7 +114,6 @@ public class ExternalCompactionUtil { ZooReader zooReader = context.getZooReader(); List groups = zooReader.getChildren(compactorGroupsPath); for (String group : groups) { -groupsAndAddresses.putIfAbsent(group, new ArrayList<>()); try { List compactors = zooReader.getChildren(compactorGroupsPath + "/" + group); for (String compactor : compactors) { @@ -124,6 +123,7 @@ public class ExternalCompactionUtil { zooReader.getChildren(compactorGroupsPath + "/" + group + "/" + compactor); if (!children.isEmpty()) { LOG.trace("Found live compactor {} ", compactor); + groupsAndAddresses.putIfAbsent(group, new ArrayList<>()); groupsAndAddresses.get(group).add(HostAndPort.fromString(compactor)); } } diff --git a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java index 21e831c3e0..9bf50cff2e 100644 --- a/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java +++ b/minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloClusterControl.java @@ -519,6 +519,10 @@ public class MiniAccumuloClusterControl implements ClusterControl { stop(server, hostname); } + public List getCompactors(String resourceGroup) { +return compactorProcesses.get(resourceGroup); + } + public List getTabletServers(String resourceGroup) { return tabletServerProcesses.get(resourceGroup); } diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java index 98222a5fa6..19723a5391 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java @@ -22,6 +22,7 @@ import static org.apache.accumulo.test.util.Wait.waitFor; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; import java.util.List; import java.util.Map; @@ -66,6 +67,7 @@ public class MemoryStarvedMajCIT extends SharedMiniClusterBase { cfg.getClusterServerConfiguration().setNumDefaultTabletServers(1); cfg.getClusterServerConfiguration().setNumDefaultScanServers(0); cfg.getClusterServerConfiguration().setNumDefaultCompactors(1); + cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "15s"); cfg.setProperty(Property.GENERAL_LOW_MEM_DETECTOR_INTERVAL, "5s"); cfg.setProperty(Property.GENERAL_LOW_MEM_DETECTOR_THRESHOLD, Double.toString(MemoryStarvedScanIT.FREE_MEMORY_THRESHOLD)); @@ -132,10 +134,23 @@ public class MemoryStarvedMajCIT extends SharedMiniClusterBase { ClientContext ctx = (ClientContext) client; - // Stop the normal compactors and start the version that will consume - // and free memory when we need it to - getCluster().getClusterControl().stopAllServers(ServerType.COMPACTOR); + // Kill the normal compactors and
(accumulo-examples) 01/01: Start point
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a commit to branch helm-chart-collab in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git commit d5031384604bddec23b67843343665534acc8887 Author: Daniel Roberts AuthorDate: Thu Feb 15 17:24:12 2024 + Start point --- helm/accumulo/.helmignore | 23 ++ helm/accumulo/Chart.yaml | 24 +++ helm/accumulo/templates/NOTES.txt | 22 ++ helm/accumulo/templates/_helpers.tpl | 62 helm/accumulo/templates/deployment.yaml| 61 helm/accumulo/templates/hpa.yaml | 28 helm/accumulo/templates/ingress.yaml | 61 helm/accumulo/templates/service.yaml | 15 helm/accumulo/templates/serviceaccount.yaml| 12 helm/accumulo/templates/tests/test-connection.yaml | 15 helm/accumulo/values.yaml | 82 ++ 11 files changed, 405 insertions(+) diff --git a/helm/accumulo/.helmignore b/helm/accumulo/.helmignore new file mode 100644 index 000..0e8a0eb --- /dev/null +++ b/helm/accumulo/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/helm/accumulo/Chart.yaml b/helm/accumulo/Chart.yaml new file mode 100644 index 000..163dd45 --- /dev/null +++ b/helm/accumulo/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: accumulo +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/helm/accumulo/templates/NOTES.txt b/helm/accumulo/templates/NOTES.txt new file mode 100644 index 000..71a6b4e --- /dev/null +++ b/helm/accumulo/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "accumulo.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "accumulo.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "accumulo.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "accumulo.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff
(accumulo-examples) branch helm-chart-collab created (now d503138)
This is an automated email from the ASF dual-hosted git repository. ddanielr pushed a change to branch helm-chart-collab in repository https://gitbox.apache.org/repos/asf/accumulo-examples.git at d503138 Start point This branch includes the following new commits: new d503138 Start point The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(accumulo-access) branch main updated: moves thread local access outside of loop (#61)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/accumulo-access.git The following commit(s) were added to refs/heads/main by this push: new d0b4423 moves thread local access outside of loop (#61) d0b4423 is described below commit d0b4423087faf78dd0e96e3042bbc12fa87d6f42 Author: Keith Turner AuthorDate: Thu Feb 15 11:06:14 2024 -0500 moves thread local access outside of loop (#61) A thread local was being accessed multiple times in a loop. The result should be the same each time, so the access was moved out of the loop to avoid any cost related to accessing a thread local. --- src/main/java/org/apache/accumulo/access/AccessEvaluatorImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/accumulo/access/AccessEvaluatorImpl.java b/src/main/java/org/apache/accumulo/access/AccessEvaluatorImpl.java index 66adb96..a73f627 100644 --- a/src/main/java/org/apache/accumulo/access/AccessEvaluatorImpl.java +++ b/src/main/java/org/apache/accumulo/access/AccessEvaluatorImpl.java @@ -196,9 +196,9 @@ final class AccessEvaluatorImpl implements AccessEvaluator { boolean evaluate(byte[] accessExpression) throws IllegalAccessExpressionException { var bytesWrapper = lookupWrappers.get(); +var tokenizer = tokenizers.get(); for (var auths : authorizedPredicates) { - var tokenizer = tokenizers.get(); tokenizer.reset(accessExpression); Predicate atp = authToken -> { bytesWrapper.set(authToken.data, authToken.start, authToken.len);
(accumulo) branch elasticity updated: Adds unit test for SetEqualityIterator (#4227)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/elasticity by this push: new b5afd3cc1c Adds unit test for SetEqualityIterator (#4227) b5afd3cc1c is described below commit b5afd3cc1cfcf05c05be79ec92be08ffae1416d7 Author: rsingh433 <74160026+rsingh...@users.noreply.github.com> AuthorDate: Thu Feb 15 10:56:25 2024 -0500 Adds unit test for SetEqualityIterator (#4227) --- .../accumulo/server/SetEqualityIteratorTest.java | 179 + 1 file changed, 179 insertions(+) diff --git a/server/base/src/test/java/org/apache/accumulo/server/SetEqualityIteratorTest.java b/server/base/src/test/java/org/apache/accumulo/server/SetEqualityIteratorTest.java new file mode 100644 index 00..3d6c11f9b8 --- /dev/null +++ b/server/base/src/test/java/org/apache/accumulo/server/SetEqualityIteratorTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.io.IOException; +import java.util.Collections; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.iteratorsImpl.system.SortedMapIterator; +import org.apache.accumulo.core.metadata.ReferencedTabletFile; +import org.apache.accumulo.core.metadata.StoredTabletFile; +import org.apache.accumulo.core.metadata.schema.DataFileValue; +import org.apache.accumulo.core.metadata.schema.MetadataSchema; +import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.server.metadata.iterators.SetEqualityIterator; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.io.Text; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class SetEqualityIteratorTest { + + private SetEqualityIterator setEqualityIterator; + private SetEqualityIterator setEqualityIteratorNoFiles; + private SetEqualityIterator setEqualityIteratorOneFile; + private SortedMapIterator sortedMapIterator; + private SortedMapIterator sortedMapIteratorNoFiles; + private SortedMapIterator sortedMapIteratorOneFile; + + private KeyExtent extent = new KeyExtent(TableId.of("5"), new Text("df"), new Text("da")); + + private StoredTabletFile file1 = + new ReferencedTabletFile(new Path("dfs://nn1/acc/tables/1/t-0001/sf1.rf")).insert(); + private StoredTabletFile file2 = + new ReferencedTabletFile(new Path("dfs://nn1/acc/tables/1/t-0001/sf2.rf")).insert(); + private StoredTabletFile file3 = + new ReferencedTabletFile(new Path("dfs://nn1/acc/tables/1/t-0001/sf3.rf")).insert(); + + @BeforeEach + public void setUp() throws IOException { + +// Create tablet metadata with no files +TabletMetadata tmNoFiles = TabletMetadata.builder(extent).putFlushId(7).build(); + +// Create tablet metadata with one file +StoredTabletFile singleFile = +new ReferencedTabletFile(new Path("dfs://nn1/acc/tables/1/t-0001/sf1.rf")).insert(); +TabletMetadata tmOneFile = TabletMetadata.builder(extent) +.putFile(singleFile, new DataFileValue(100, 50)).putFlushId(8).build(); + +// Create tablet metadata with multiple files +TabletMetadata tmMultipleFiles = TabletMetadata.builder(extent) +.putFile(file1, new DataFileValue(0, 0)).putFile(file2, new DataFileValue(555, 23)) +.putFile(file3, new DataFileValue(234, 13)).putFlushId(6).build(); + +var extent2 = new KeyExtent(extent.tableId(), null, extent.endRow()); +// create another tablet metadata using extent2 w/ diff files and add it to sortedMap. This +// will add another row to the test data which ensures that iterator does not
(accumulo) branch elasticity updated: Bug fix from PR #4247 (#4265)
This is an automated email from the ASF dual-hosted git repository. kturner pushed a commit to branch elasticity in repository https://gitbox.apache.org/repos/asf/accumulo.git The following commit(s) were added to refs/heads/elasticity by this push: new 10d0824fac Bug fix from PR #4247 (#4265) 10d0824fac is described below commit 10d0824fac42faefba3305d4dcf0dc676dffb54d Author: Kevin Rathbun <43969518+kevinrr...@users.noreply.github.com> AuthorDate: Thu Feb 15 09:55:28 2024 -0500 Bug fix from PR #4247 (#4265) --- .../org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java index 892a507530..a193357597 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/AmpleConditionalWriterIT.java @@ -631,7 +631,7 @@ public class AmpleConditionalWriterIT extends AccumuloClusterHarness { * * * { - * "txid": "FATE:META:123456", + * "fateId": "FATE:META:123456", * "selAll": true, * "files": ["/path/to/file1.rf", "/path/to/file2.rf"] * } @@ -640,7 +640,7 @@ public class AmpleConditionalWriterIT extends AccumuloClusterHarness { public static String createSelectedFilesJson(FateId fateId, boolean selAll, Collection paths) { String filesJsonArray = GSON.get().toJson(paths); -return ("{'txid':'" + fateId + "','selAll':" + selAll + ",'files':" + filesJsonArray + "}") +return ("{'fateId':'" + fateId + "','selAll':" + selAll + ",'files':" + filesJsonArray + "}") .replace('\'', '\"'); }