This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit 44adbba68d676ee2ed0d523a73138f0759af4c9e Author: Michael Blow <[email protected]> AuthorDate: Sun Oct 12 09:37:04 2025 -0400 [NO ISSUE][*DB][STO] Increased log brevity in cloud storage Ext-ref: MB-68946 Change-Id: I37332782d583d5c9daaf8b58dad7c4a1a2992bd6 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20483 Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Michael Blow <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> --- .../asterix/cloud/AbstractCloudIOManager.java | 10 +++--- .../cloud/lazy/accessor/InitialCloudAccessor.java | 5 ++- .../lazy/accessor/ReplaceableCloudAccessor.java | 5 +-- .../lazy/accessor/SelectiveCloudAccessor.java | 4 +-- .../apache/asterix/cloud/util/CloudFileUtil.java | 1 - .../apache/asterix/common/utils/Partitions.java | 38 ++++++++++++++++++++-- 6 files changed, 47 insertions(+), 16 deletions(-) diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java index 526af2df1c..7712f08653 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java @@ -31,7 +31,6 @@ import java.nio.file.FileStore; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.function.Predicate; @@ -50,6 +49,7 @@ import org.apache.asterix.common.cloud.IPartitionBootstrapper; import org.apache.asterix.common.config.CloudProperties; import org.apache.asterix.common.metadata.MetadataConstants; import org.apache.asterix.common.transactions.IRecoveryManager.SystemState; +import org.apache.asterix.common.utils.Partitions; import org.apache.asterix.common.utils.StoragePathUtil; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; @@ -82,12 +82,11 @@ public abstract class AbstractCloudIOManager extends IOManager implements IParti protected final ICloudGuardian guardian; protected final IWriteBufferProvider writeBufferProvider; protected final String bucket; - protected final Set<Integer> partitions; + protected final Partitions partitions; protected final List<FileReference> partitionPaths; protected final IOManager localIoManager; protected final INamespacePathResolver nsPathResolver; private final List<FileStore> drivePaths; - private final String storageScheme; public AbstractCloudIOManager(IOManager ioManager, CloudProperties cloudProperties, INamespacePathResolver nsPathResolver, ICloudGuardian guardian) throws HyracksDataException { @@ -99,11 +98,10 @@ public abstract class AbstractCloudIOManager extends IOManager implements IParti this.guardian = guardian; int numOfThreads = getIODevices().size() * getIOParallelism(); writeBufferProvider = new WriteBufferProvider(numOfThreads, cloudClient.getWriteBufferSize()); - partitions = new HashSet<>(); + partitions = new Partitions(); partitionPaths = new ArrayList<>(); this.localIoManager = ioManager; drivePaths = PhysicalDrive.getDrivePaths(ioDevices); - storageScheme = cloudProperties.getStorageScheme(); } /* @@ -179,7 +177,7 @@ public abstract class AbstractCloudIOManager extends IOManager implements IParti } } } else { - LOGGER.info("Cleaning node partitions..."); + LOGGER.info("Cleaning node partitions {}...", partitions); for (FileReference partitionPath : partitionPaths) { CloudFileUtil.cleanDirectoryFiles(localIoManager, cloudFiles, partitionPath); } diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java index 8c3321ad86..5070d40bce 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java @@ -18,10 +18,9 @@ */ package org.apache.asterix.cloud.lazy.accessor; -import java.util.Collections; - import org.apache.asterix.cloud.clients.ICloudClient; import org.apache.asterix.cloud.lazy.NoOpParallelCacher; +import org.apache.asterix.common.utils.Partitions; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.control.nc.io.IOManager; @@ -35,7 +34,7 @@ public class InitialCloudAccessor extends ReplaceableCloudAccessor { }; public InitialCloudAccessor(ICloudClient cloudClient, String bucket, IOManager localIoManager) { - super(cloudClient, bucket, localIoManager, Collections.emptySet(), NO_OP_REPLACER, NoOpParallelCacher.INSTANCE); + super(cloudClient, bucket, localIoManager, Partitions.empty(), NO_OP_REPLACER, NoOpParallelCacher.INSTANCE); } @Override diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java index 91046a1a06..8dbf605608 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java @@ -28,6 +28,7 @@ import org.apache.asterix.cloud.bulk.IBulkOperationCallBack; import org.apache.asterix.cloud.clients.CloudFile; import org.apache.asterix.cloud.clients.ICloudClient; import org.apache.asterix.cloud.lazy.IParallelCacher; +import org.apache.asterix.common.utils.Partitions; import org.apache.asterix.common.utils.StorageConstants; import org.apache.asterix.common.utils.StoragePathUtil; import org.apache.hyracks.api.exceptions.HyracksDataException; @@ -42,13 +43,13 @@ import org.apache.logging.log4j.Logger; */ public class ReplaceableCloudAccessor extends AbstractLazyAccessor { private static final Logger LOGGER = LogManager.getLogger(); - private final Set<Integer> partitions; + private final Partitions partitions; private final ILazyAccessorReplacer replacer; protected final IParallelCacher cacher; private final IBulkOperationCallBack deleteCallBack; public ReplaceableCloudAccessor(ICloudClient cloudClient, String bucket, IOManager localIoManager, - Set<Integer> partitions, ILazyAccessorReplacer replacer, IParallelCacher cacher) { + Partitions partitions, ILazyAccessorReplacer replacer, IParallelCacher cacher) { super(cloudClient, bucket, localIoManager); this.partitions = partitions; this.replacer = replacer; diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java index 56eaeeb08b..68c0556fa3 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java @@ -21,12 +21,12 @@ package org.apache.asterix.cloud.lazy.accessor; import static org.apache.asterix.cloud.util.CloudFileUtil.DATA_FILTER; import java.util.Collection; -import java.util.Set; import org.apache.asterix.cloud.UncachedFileReference; import org.apache.asterix.cloud.clients.ICloudClient; import org.apache.asterix.cloud.lazy.IParallelCacher; import org.apache.asterix.cloud.lazy.filesystem.IHolePuncher; +import org.apache.asterix.common.utils.Partitions; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.api.io.FileReference; import org.apache.hyracks.api.io.IFileHandle; @@ -37,7 +37,7 @@ public class SelectiveCloudAccessor extends ReplaceableCloudAccessor { private final IHolePuncher puncher; public SelectiveCloudAccessor(ICloudClient cloudClient, String bucket, IOManager localIoManager, - Set<Integer> partitions, IHolePuncher puncher, IParallelCacher cacher) { + Partitions partitions, IHolePuncher puncher, IParallelCacher cacher) { super(cloudClient, bucket, localIoManager, partitions, InitialCloudAccessor.NO_OP_REPLACER, cacher); this.puncher = puncher; } diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java index 7befacf027..fd37e61c04 100644 --- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java +++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java @@ -49,7 +49,6 @@ public class CloudFileUtil { // First get the set of local files Set<FileReference> localFiles = ioManager.list(partitionPath); Iterator<FileReference> localFilesIter = localFiles.iterator(); - LOGGER.info("Cleaning partition {}.", partitionPath.getRelativePath()); // Reconcile local files and cloud files while (localFilesIter.hasNext()) { diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java index 45aaa49150..cc93b916cc 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java @@ -20,9 +20,13 @@ package org.apache.asterix.common.utils; import java.io.Serial; import java.io.Serializable; +import java.util.Set; + +import org.apache.commons.lang3.mutable.MutableBoolean; import it.unimi.dsi.fastutil.ints.IntIterator; import it.unimi.dsi.fastutil.ints.IntSortedSet; +import it.unimi.dsi.fastutil.ints.IntSortedSets; /** * A specialized bit set for storing partition IDs. @@ -36,21 +40,29 @@ public class Partitions implements Serializable { private static final long serialVersionUID = 1L; private static final int MAX_PARTITION_ID = Integer.MAX_VALUE - 1; private static final int MIN_PARTITION_ID = -1; + private static final Partitions EMPTY = new Partitions(IntSortedSets.EMPTY_SET); private static final short MINUS_ONE = (short) -1; private final IntSortedSet delegate; public Partitions() { - delegate = new IntSortedBitSet(); + this(new IntSortedBitSet()); } public Partitions(int initialMaxValue) { - delegate = new IntSortedBitSet(initialMaxValue + 1); + this(new IntSortedBitSet(initialMaxValue + 1)); } public Partitions(IntSortedSet delegate) { this.delegate = delegate; } + /** + * Returns an unmodifiable empty Partitions instance. + */ + public static Partitions empty() { + return EMPTY; + } + /** * Adds a partition ID to the set. Partition ID must be in the range [-1, 32766]. * @param k the partition ID to add @@ -72,6 +84,27 @@ public class Partitions implements Serializable { delegate.clear(); } + public boolean addAll(Set<Integer> activePartitions) { + MutableBoolean retval = new MutableBoolean(); + activePartitions.forEach(p -> { + if (add(p)) { + retval.setTrue(); + } + }); + return retval.booleanValue(); + } + + public int size() { + return delegate.size(); + } + + public boolean contains(int partitionNum) { + if (partitionNum > MAX_PARTITION_ID || partitionNum < MIN_PARTITION_ID) { + return false; + } + return delegate.contains((short) (partitionNum + 1)); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -107,4 +140,5 @@ public class Partitions implements Serializable { builder.append(']'); return builder.toString(); } + }
