>From Hussain Towaileb <[email protected]>:
Hussain Towaileb has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18656 )
Change subject: [WIP] Add capability to calculate collection size
......................................................................
[WIP] Add capability to calculate collection size
Ext-ref: MB-59119
Change-Id: Ia9ea1a1eee3d931fe74f26d739110834f24031e0
---
M
hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
M
hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
M
asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
3 files changed, 74 insertions(+), 8 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/56/18656/1
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index c005253..a006dec 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -30,6 +30,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.function.Predicate;
import org.apache.asterix.cloud.bulk.DeleteBulkCloudOperation;
import org.apache.asterix.cloud.bulk.NoOpDeleteBulkCallBack;
@@ -68,6 +69,8 @@
public abstract class AbstractCloudIOManager extends IOManager implements
IPartitionBootstrapper, ICloudIOManager {
private static final Logger LOGGER = LogManager.getLogger();
private static final byte[] EMPTY_FILE_BYTES = "empty".getBytes();
+ private static final Predicate<String> NO_OP_LIST_FILES_FILTER = (path) ->
true;
+
protected final ICloudClient cloudClient;
protected final ICloudGuardian guardian;
protected final IWriteBufferProvider writeBufferProvider;
@@ -376,7 +379,7 @@
public final JsonNode listAsJson(ObjectMapper objectMapper) {
ArrayNode objectsInfo = objectMapper.createArrayNode();
try {
- List<CloudFile> allFiles = list();
+ List<CloudFile> allFiles = list(NO_OP_LIST_FILES_FILTER);
allFiles.sort((x, y) ->
String.CASE_INSENSITIVE_ORDER.compare(x.getPath(), y.getPath()));
for (CloudFile file : allFiles) {
ObjectNode objectInfo = objectsInfo.addObject();
@@ -393,7 +396,7 @@
}
}
- private List<CloudFile> list() {
+ private List<CloudFile> list(Predicate<String> filter) {
List<CloudFile> allFiles = new ArrayList<>();
// get cached files (read from disk)
for (IODeviceHandle deviceHandle : getIODevices()) {
@@ -409,7 +412,9 @@
for (FileReference fileReference : deviceFiles) {
try {
- allFiles.add(CloudFile.of(fileReference.getRelativePath(),
fileReference.getFile().length()));
+ if (filter.test(fileReference.getRelativePath())) {
+
allFiles.add(CloudFile.of(fileReference.getRelativePath(),
fileReference.getFile().length()));
+ }
} catch (Throwable th) {
LOGGER.warn("Encountered issue for local storage file {}",
fileReference.getRelativePath(), th);
}
@@ -418,7 +423,9 @@
// get uncached files from uncached files tracker
for (UncachedFileReference uncachedFile : getUncachedFiles()) {
- allFiles.add(CloudFile.of(uncachedFile.getRelativePath(),
uncachedFile.getSize()));
+ if (filter.test(uncachedFile.getRelativePath())) {
+ allFiles.add(CloudFile.of(uncachedFile.getRelativePath(),
uncachedFile.getSize()));
+ }
}
return allFiles;
}
@@ -468,10 +475,46 @@
}
public long getTotalRemoteStorageSizeForNodeBytes() {
- long size = 0;
- for (CloudFile file : list()) {
- size += file.getSize();
+ return getResourceTotalSize(NO_OP_LIST_FILES_FILTER);
+ }
+
+ @Override
+ public long getCollectionSize(String fqn) {
+ return getResourceTotalSize(path -> path.contains(fqn));
+ }
+
+ private long getResourceTotalSize(Predicate<String> relativePathFilter) {
+ long totalSize = 0;
+
+ // get cached files (read from disk)
+ for (IODeviceHandle deviceHandle : getIODevices()) {
+ FileReference storageRoot =
deviceHandle.createFileRef(STORAGE_ROOT_DIR_NAME);
+
+ Set<FileReference> deviceFiles;
+ try {
+ deviceFiles = localIoManager.list(storageRoot,
IoUtil.NO_OP_FILTER);
+ } catch (Throwable th) {
+ LOGGER.warn("Failed to get local storage files for root {}",
storageRoot.getRelativePath(), th);
+ continue;
+ }
+
+ for (FileReference fileReference : deviceFiles) {
+ try {
+ if
(relativePathFilter.test(fileReference.getRelativePath())) {
+ totalSize += fileReference.getFile().length();
+ }
+ } catch (Throwable th) {
+ LOGGER.warn("Encountered issue for local storage file {}",
fileReference.getRelativePath(), th);
+ }
+ }
}
- return size;
+
+ // get uncached files from uncached files tracker
+ for (UncachedFileReference uncachedFile : getUncachedFiles()) {
+ if (relativePathFilter.test(uncachedFile.getRelativePath())) {
+ totalSize += uncachedFile.getSize();
+ }
+ }
+ return totalSize;
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
index a6520c6..beca30d 100644
---
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/io/IIOManager.java
@@ -156,6 +156,14 @@
long getSize(FileReference fileReference) throws HyracksDataException;
/**
+ * Gets the size of the provided collection
+ *
+ * @param fqn fully qualified name of the collection
+ * @return resource size
+ */
+ long getCollectionSize(String fqn) throws HyracksDataException;
+
+ /**
* Returns a new write channel
*
* @param fileHandle handle of the opened file
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
index b35111e..a79a25c 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java
@@ -601,6 +601,11 @@
((AbstractBulkOperation) bulkOperation).performOperation();
}
+ public long getCollectionSize(String fqn) {
+ // TODO(htowaileb): Do nothing for now
+ return 0;
+ }
+
public void setSpaceMaker(IDiskSpaceMaker spaceMaker) {
this.spaceMaker = spaceMaker;
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18656
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: goldfish
Gerrit-Change-Id: Ia9ea1a1eee3d931fe74f26d739110834f24031e0
Gerrit-Change-Number: 18656
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <[email protected]>
Gerrit-MessageType: newchange
