cshannon commented on code in PR #5247: URL: https://github.com/apache/accumulo/pull/5247#discussion_r1912239650
########## server/base/src/main/java/org/apache/accumulo/server/metadata/iterators/TabletMetadataCheckIterator.java: ########## @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.metadata.iterators; + +import static org.apache.accumulo.core.util.LazySingletons.GSON; +import static org.apache.accumulo.server.metadata.iterators.ColumnFamilyTransformationIterator.getTabletRow; + +import java.io.IOException; +import java.util.Collection; +import java.util.EnumSet; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Set; + +import org.apache.accumulo.core.classloader.ClassLoaderUtil; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.data.ByteSequence; +import org.apache.accumulo.core.data.Condition; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.iterators.IteratorAdapter; +import org.apache.accumulo.core.iterators.IteratorEnvironment; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.metadata.schema.TabletMetadataCheck; +import org.apache.accumulo.server.metadata.ConditionalTabletMutatorImpl; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Preconditions; + +public class TabletMetadataCheckIterator implements SortedKeyValueIterator<Key,Value> { + + private static final Logger log = LoggerFactory.getLogger(TabletMetadataCheckIterator.class); + + private TabletMetadataCheck check; + private KeyExtent expectedExtent; + + private Key startKey; + private Value topValue; + private SortedKeyValueIterator<Key,Value> source; + + private static final String CHECK_CLASS_KEY = "checkClass"; + private static final String CHECK_DATA_KEY = "checkData"; + private static final String EXTENT_KEY = "checkExtent"; + private static final String SUCCESS = "success"; + + @Override + public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, + IteratorEnvironment env) throws IOException { + Preconditions.checkState(check == null && expectedExtent == null && startKey == null); + try { + String className = Objects.requireNonNull(options.get(CHECK_CLASS_KEY)); + String checkData = Objects.requireNonNull(options.get(CHECK_DATA_KEY)); + log.trace("Instantiating class {} using {}", className, checkData); + Class<? extends TabletMetadataCheck> clazz = + ClassLoaderUtil.loadClass(null, className, TabletMetadataCheck.class); + check = GSON.get().fromJson(checkData, clazz); + expectedExtent = KeyExtent.fromBase64(options.get(EXTENT_KEY)); + this.source = source; + } catch (ReflectiveOperationException e) { + throw new IllegalStateException(e); Review Comment: Would it be useful to add any logging here for a ClassNotFoundException if the impl isn't on the server classpath? It may not be necessary if this exception will be logged higher up when the IllegalStateException is thrown. ########## server/base/src/main/java/org/apache/accumulo/server/metadata/iterators/TabletMetadataCheckIterator.java: ########## @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.server.metadata.iterators; + +import static org.apache.accumulo.core.util.LazySingletons.GSON; +import static org.apache.accumulo.server.metadata.iterators.ColumnFamilyTransformationIterator.getTabletRow; + +import java.io.IOException; +import java.util.Collection; +import java.util.EnumSet; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Set; + +import org.apache.accumulo.core.classloader.ClassLoaderUtil; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.data.ByteSequence; +import org.apache.accumulo.core.data.Condition; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.dataImpl.KeyExtent; +import org.apache.accumulo.core.iterators.IteratorAdapter; +import org.apache.accumulo.core.iterators.IteratorEnvironment; +import org.apache.accumulo.core.iterators.SortedKeyValueIterator; +import org.apache.accumulo.core.metadata.schema.TabletMetadata; +import org.apache.accumulo.core.metadata.schema.TabletMetadataCheck; +import org.apache.accumulo.server.metadata.ConditionalTabletMutatorImpl; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.base.Preconditions; + +public class TabletMetadataCheckIterator implements SortedKeyValueIterator<Key,Value> { + + private static final Logger log = LoggerFactory.getLogger(TabletMetadataCheckIterator.class); + + private TabletMetadataCheck check; + private KeyExtent expectedExtent; + + private Key startKey; + private Value topValue; + private SortedKeyValueIterator<Key,Value> source; + + private static final String CHECK_CLASS_KEY = "checkClass"; + private static final String CHECK_DATA_KEY = "checkData"; + private static final String EXTENT_KEY = "checkExtent"; + private static final String SUCCESS = "success"; + + @Override + public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> options, + IteratorEnvironment env) throws IOException { + Preconditions.checkState(check == null && expectedExtent == null && startKey == null); + try { + String className = Objects.requireNonNull(options.get(CHECK_CLASS_KEY)); + String checkData = Objects.requireNonNull(options.get(CHECK_DATA_KEY)); + log.trace("Instantiating class {} using {}", className, checkData); + Class<? extends TabletMetadataCheck> clazz = + ClassLoaderUtil.loadClass(null, className, TabletMetadataCheck.class); + check = GSON.get().fromJson(checkData, clazz); + expectedExtent = KeyExtent.fromBase64(options.get(EXTENT_KEY)); + this.source = source; + } catch (ReflectiveOperationException e) { + throw new IllegalStateException(e); + } + } + + @Override + public void seek(Range range, Collection<ByteSequence> columnFamilies, boolean inclusive) + throws IOException { + Preconditions.checkState(check != null && expectedExtent != null); + + this.startKey = null; + + Text tabletRow = getTabletRow(range); + + var expectedMetaRow = expectedExtent.toMetaRow(); + Preconditions.checkState(tabletRow.equals(expectedMetaRow), "Tablet row mismatch %s %s", + tabletRow, expectedMetaRow); + + var colsToRead = check.columnsToRead(); + // TODO actually use columns in seek and TabletMetadata construction Review Comment: I was actually just getting ready to ask you if you planned to do this now or in a future PR ########## core/src/main/java/org/apache/accumulo/core/metadata/schema/TabletMetadataCheck.java: ########## @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.core.metadata.schema; + +import java.util.Collections; +import java.util.EnumSet; +import java.util.Set; + +/** + * This interface facilitates atomic checks of tablet metadata prior to updating tablet metadata. + * The way it is intended to be used is the following. + * <ol> + * <li>On the client side a TabletMetadataCheck object is created and passed to Ample</li> Review Comment: I'm wondering if there is any good way to specifically validate that an implementation can be correctly serialized by gson, especially because we've configured our gson to not allow unsafe usage (so we require a default constructor and not using final etc). I'm guessing the only way is just to to make sure there are tests that use the new check in a conditional mutation and verify it works which is probably fine as we should be writing tests anyways. ########## server/manager/src/main/java/org/apache/accumulo/manager/compaction/coordinator/CompactionCoordinator.java: ########## @@ -575,97 +496,50 @@ private ReserveCompactionTask(CompactionJobQueues.MetaJob metaJob, String compac @Override public CompactionMetadata get() { - try { - var metaJob = metaJobRef.get(); - if (metaJob == null) { - LOG.warn("Compaction reservation request for {} {} was garbage collected.", - compactorAddress, externalCompactionId); - return null; - } - - var tabletMetadata = metaJob.getTabletMetadata(); - - var jobFiles = metaJob.getJob().getFiles().stream() - .map(CompactableFileImpl::toStoredTabletFile).collect(Collectors.toSet()); - - Retry retry = Retry.builder().maxRetries(5).retryAfter(Duration.ofMillis(100)) - .incrementBy(Duration.ofMillis(100)).maxWait(Duration.ofSeconds(10)).backOffFactor(1.5) - .logInterval(Duration.ofMinutes(3)).createRetry(); - - while (retry.canRetry()) { - try (var tabletsMutator = ctx.getAmple().conditionallyMutateTablets()) { - var extent = metaJob.getTabletMetadata().getExtent(); - - if (!canReserveCompaction(tabletMetadata, metaJob.getJob().getKind(), jobFiles, ctx, - manager.getSteadyTime())) { - return null; - } - - var ecm = createExternalCompactionMetadata(metaJob.getJob(), jobFiles, tabletMetadata, - compactorAddress, externalCompactionId); - - // any data that is read from the tablet to make a decision about if it can compact or - // not - // must be checked for changes in the conditional mutation. - var tabletMutator = tabletsMutator.mutateTablet(extent).requireAbsentOperation() - .requireFiles(jobFiles).requireNotCompacting(jobFiles); - if (metaJob.getJob().getKind() == CompactionKind.SYSTEM) { - // For system compactions the user compaction requested column is examined when - // deciding - // if a compaction can start so need to check for changes to this column. - tabletMutator.requireSame(tabletMetadata, SELECTED, USER_COMPACTION_REQUESTED); - } else { - tabletMutator.requireSame(tabletMetadata, SELECTED); - } - - if (metaJob.getJob().getKind() == CompactionKind.SYSTEM) { - var selectedFiles = tabletMetadata.getSelectedFiles(); - var reserved = - getFilesReservedBySelection(tabletMetadata, manager.getSteadyTime(), ctx); - - // If there is a selectedFiles column, and the reserved set is empty this means that - // either no user jobs were completed yet or the selection expiration time has passed - // so the column is eligible to be deleted so a system job can run instead - if (selectedFiles != null && reserved.isEmpty() - && !Collections.disjoint(jobFiles, selectedFiles.getFiles())) { - LOG.debug("Deleting user compaction selected files for {} {}", extent, - externalCompactionId); - tabletMutator.deleteSelectedFiles(); - } - } + if (ctx.getTableState(rcJob.getExtent().tableId()) != TableState.ONLINE) { + return null; + } - tabletMutator.putExternalCompaction(externalCompactionId, ecm); - tabletMutator.submit( - tm -> tm.getExternalCompactions().containsKey(externalCompactionId), - () -> "compaction reservation"); + try { + try (var tabletsMutator = ctx.getAmple().conditionallyMutateTablets()) { Review Comment: the changes here are really nice, it's definitely a lot simpler. I think it would make moving the reservation to the Compactor easier as well if we implement #4978 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
