>From Michael Blow <[email protected]>:
Michael Blow has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20483?usp=email )
Change subject: [NO ISSUE][*DB][STO] Increased log brevity in cloud storage
......................................................................
[NO ISSUE][*DB][STO] Increased log brevity in cloud storage
Change-Id: I37332782d583d5c9daaf8b58dad7c4a1a2992bd6
---
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
6 files changed, 55 insertions(+), 18 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/83/20483/1
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index 526af2d..7712f08 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -31,7 +31,6 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.Predicate;
@@ -50,6 +49,7 @@
import org.apache.asterix.common.config.CloudProperties;
import org.apache.asterix.common.metadata.MetadataConstants;
import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
+import org.apache.asterix.common.utils.Partitions;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
@@ -82,12 +82,11 @@
protected final ICloudGuardian guardian;
protected final IWriteBufferProvider writeBufferProvider;
protected final String bucket;
- protected final Set<Integer> partitions;
+ protected final Partitions partitions;
protected final List<FileReference> partitionPaths;
protected final IOManager localIoManager;
protected final INamespacePathResolver nsPathResolver;
private final List<FileStore> drivePaths;
- private final String storageScheme;
public AbstractCloudIOManager(IOManager ioManager, CloudProperties
cloudProperties,
INamespacePathResolver nsPathResolver, ICloudGuardian guardian)
throws HyracksDataException {
@@ -99,11 +98,10 @@
this.guardian = guardian;
int numOfThreads = getIODevices().size() * getIOParallelism();
writeBufferProvider = new WriteBufferProvider(numOfThreads,
cloudClient.getWriteBufferSize());
- partitions = new HashSet<>();
+ partitions = new Partitions();
partitionPaths = new ArrayList<>();
this.localIoManager = ioManager;
drivePaths = PhysicalDrive.getDrivePaths(ioDevices);
- storageScheme = cloudProperties.getStorageScheme();
}
/*
@@ -179,7 +177,7 @@
}
}
} else {
- LOGGER.info("Cleaning node partitions...");
+ LOGGER.info("Cleaning node partitions {}...", partitions);
for (FileReference partitionPath : partitionPaths) {
CloudFileUtil.cleanDirectoryFiles(localIoManager, cloudFiles,
partitionPath);
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java
index 8c3321a..5070d40 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java
@@ -18,10 +18,9 @@
*/
package org.apache.asterix.cloud.lazy.accessor;
-import java.util.Collections;
-
import org.apache.asterix.cloud.clients.ICloudClient;
import org.apache.asterix.cloud.lazy.NoOpParallelCacher;
+import org.apache.asterix.common.utils.Partitions;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.control.nc.io.IOManager;
@@ -35,7 +34,7 @@
};
public InitialCloudAccessor(ICloudClient cloudClient, String bucket,
IOManager localIoManager) {
- super(cloudClient, bucket, localIoManager, Collections.emptySet(),
NO_OP_REPLACER, NoOpParallelCacher.INSTANCE);
+ super(cloudClient, bucket, localIoManager, Partitions.empty(),
NO_OP_REPLACER, NoOpParallelCacher.INSTANCE);
}
@Override
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java
index 91046a1..8dbf605 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java
@@ -28,6 +28,7 @@
import org.apache.asterix.cloud.clients.CloudFile;
import org.apache.asterix.cloud.clients.ICloudClient;
import org.apache.asterix.cloud.lazy.IParallelCacher;
+import org.apache.asterix.common.utils.Partitions;
import org.apache.asterix.common.utils.StorageConstants;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -42,13 +43,13 @@
*/
public class ReplaceableCloudAccessor extends AbstractLazyAccessor {
private static final Logger LOGGER = LogManager.getLogger();
- private final Set<Integer> partitions;
+ private final Partitions partitions;
private final ILazyAccessorReplacer replacer;
protected final IParallelCacher cacher;
private final IBulkOperationCallBack deleteCallBack;
public ReplaceableCloudAccessor(ICloudClient cloudClient, String bucket,
IOManager localIoManager,
- Set<Integer> partitions, ILazyAccessorReplacer replacer,
IParallelCacher cacher) {
+ Partitions partitions, ILazyAccessorReplacer replacer,
IParallelCacher cacher) {
super(cloudClient, bucket, localIoManager);
this.partitions = partitions;
this.replacer = replacer;
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java
index 56eaeeb..68c0556 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java
@@ -21,12 +21,12 @@
import static org.apache.asterix.cloud.util.CloudFileUtil.DATA_FILTER;
import java.util.Collection;
-import java.util.Set;
import org.apache.asterix.cloud.UncachedFileReference;
import org.apache.asterix.cloud.clients.ICloudClient;
import org.apache.asterix.cloud.lazy.IParallelCacher;
import org.apache.asterix.cloud.lazy.filesystem.IHolePuncher;
+import org.apache.asterix.common.utils.Partitions;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IFileHandle;
@@ -37,7 +37,7 @@
private final IHolePuncher puncher;
public SelectiveCloudAccessor(ICloudClient cloudClient, String bucket,
IOManager localIoManager,
- Set<Integer> partitions, IHolePuncher puncher, IParallelCacher
cacher) {
+ Partitions partitions, IHolePuncher puncher, IParallelCacher
cacher) {
super(cloudClient, bucket, localIoManager, partitions,
InitialCloudAccessor.NO_OP_REPLACER, cacher);
this.puncher = puncher;
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
index 7befacf..fd37e61 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
@@ -49,7 +49,6 @@
// First get the set of local files
Set<FileReference> localFiles = ioManager.list(partitionPath);
Iterator<FileReference> localFilesIter = localFiles.iterator();
- LOGGER.info("Cleaning partition {}.", partitionPath.getRelativePath());
// Reconcile local files and cloud files
while (localFilesIter.hasNext()) {
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
index 6d62f64..b40a8e2 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
@@ -20,6 +20,12 @@
import java.io.Serial;
import java.io.Serializable;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.MutableBoolean;
+
+import it.unimi.dsi.fastutil.shorts.ShortSortedSet;
+import it.unimi.dsi.fastutil.shorts.ShortSortedSets;
/**
* A specialized bit set for storing partition IDs.
@@ -33,14 +39,26 @@
private static final long serialVersionUID = 1L;
private static final int MAX_PARTITION_ID = Short.MAX_VALUE - 1;
private static final int MIN_PARTITION_ID = -1;
- private final ShortSortedBitSet delegate;
+ private static final Partitions EMPTY = new
Partitions(ShortSortedSets.EMPTY_SET);
+ private final ShortSortedSet delegate;
public Partitions() {
- delegate = new ShortSortedBitSet();
+ this(new ShortSortedBitSet());
}
public Partitions(int initialMaxValue) {
- delegate = new ShortSortedBitSet(initialMaxValue + 1);
+ this(new ShortSortedBitSet(initialMaxValue + 1));
+ }
+
+ public Partitions(ShortSortedSet delegate) {
+ this.delegate = delegate;
+ }
+
+ /**
+ * Returns an unmodifiable empty Partitions instance.
+ */
+ public static Partitions empty() {
+ return EMPTY;
}
/**
@@ -64,6 +82,27 @@
delegate.clear();
}
+ public boolean addAll(Set<Integer> activePartitions) {
+ MutableBoolean retval = new MutableBoolean();
+ activePartitions.forEach(p -> {
+ if (add(p)) {
+ retval.setTrue();
+ }
+ });
+ return retval.booleanValue();
+ }
+
+ public int size() {
+ return delegate.size();
+ }
+
+ public boolean contains(int partitionNum) {
+ if (partitionNum > MAX_PARTITION_ID || partitionNum <
MIN_PARTITION_ID) {
+ return false;
+ }
+ return delegate.contains((short) (partitionNum + 1));
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -82,6 +121,7 @@
@Override
public String toString() {
- return ShortUtil.toCompactString(delegate.iterator(), (short)
MIN_PARTITION_ID);
+ return ShortUtil.toCompactString(delegate.iterator(), (short) -1);
}
+
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20483?usp=email
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings?usp=email
Gerrit-MessageType: newchange
Gerrit-Project: asterixdb
Gerrit-Branch: phoenix
Gerrit-Change-Id: I37332782d583d5c9daaf8b58dad7c4a1a2992bd6
Gerrit-Change-Number: 20483
Gerrit-PatchSet: 1
Gerrit-Owner: Michael Blow <[email protected]>
