This is an automated email from the ASF dual-hosted git repository.
wyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 25edb259ce [MULTIPLE] Metrics and incomplete downloads fixes
25edb259ce is described below
commit 25edb259ce11afc2a30c1da39fb01524a4541d68
Author: Wail Alkowaileet <[email protected]>
AuthorDate: Mon Jul 1 10:55:09 2024 -0700
[MULTIPLE] Metrics and incomplete downloads fixes
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- ASTERIXDB-3449: 'metrics' object can miss commas
- ASTERIXDB-3448: Delete partially downloaded files
on bootstrap
Change-Id: I9796906ec2c68aed323c91ed673aba6833f257dd
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18409
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Wail Alkowaileet <[email protected]>
Reviewed-by: Ali Alsuliman <[email protected]>
---
.../asterix/app/result/fields/MetricsPrinter.java | 4 ++--
.../apache/asterix/cloud/LazyCloudIOManager.java | 2 +-
.../apache/asterix/cloud/clients/CloudFile.java | 23 ++++++++++++++++++++--
.../apache/asterix/cloud/util/CloudFileUtil.java | 12 +++++++----
4 files changed, 32 insertions(+), 9 deletions(-)
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
index a8a3135ef5..1e6553334c 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
@@ -101,7 +101,7 @@ public class MetricsPrinter implements
IResponseFieldPrinter {
pw.print("\n");
pw.print("\t");
ResultUtil.printField(pw,
Metrics.BUFFERCACHE_PAGEREAD_COUNT.str(), metrics.getBufferCachePageReadCount(),
- hasWarnings || hasErrors);
+ hasWarnings || hasErrors || madeCloudReadRequests);
pw.print("\n");
}
if (madeCloudReadRequests) {
@@ -115,7 +115,7 @@ public class MetricsPrinter implements
IResponseFieldPrinter {
pw.print("\n");
pw.print("\t");
ResultUtil.printField(pw,
Metrics.REMOTE_PAGES_PERSISTED_COUNT.str(),
metrics.getCloudPagesPersistedCount(),
- true);
+ hasWarnings || hasErrors);
pw.print("\n");
}
if (hasWarnings) {
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
index f8cf2c34e2..35b7255d25 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LazyCloudIOManager.java
@@ -113,7 +113,7 @@ final class LazyCloudIOManager extends
AbstractCloudIOManager {
}
}
- // Keep uncached files list (i.e., files exists in cloud only)
+ // Keep uncached files list (i.e., files exist in cloud only)
cloudFiles.removeAll(localFiles);
int remainingUncachedFiles = cloudFiles.size();
boolean canReplaceAccessor = replacer !=
InitialCloudAccessor.NO_OP_REPLACER;
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudFile.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudFile.java
index e1cf135915..50edd7b99e 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudFile.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/CloudFile.java
@@ -18,7 +18,12 @@
*/
package org.apache.asterix.cloud.clients;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
public final class CloudFile {
+ private static final long IGNORED_SIZE = -1;
private final String path;
private final long size;
@@ -47,7 +52,7 @@ public final class CloudFile {
}
CloudFile other = (CloudFile) obj;
- return path.equals(other.path);
+ return path.equals(other.path) && compareSize(other.size);
}
@Override
@@ -55,11 +60,25 @@ public final class CloudFile {
return path;
}
+ private boolean compareSize(long otherSize) {
+ // Compare sizes iff both sizes are not ignored
+ return size == otherSize || size == IGNORED_SIZE || otherSize ==
IGNORED_SIZE;
+ }
+
public static CloudFile of(String path, long size) {
return new CloudFile(path, size);
}
public static CloudFile of(String path) {
- return new CloudFile(path, 0);
+ return new CloudFile(path, IGNORED_SIZE);
+ }
+
+ public static Map<String, CloudFile> toMap(Set<CloudFile> cloudFiles) {
+ Map<String, CloudFile> map = new HashMap<>();
+ for (CloudFile cloudFile : cloudFiles) {
+ map.put(cloudFile.getPath(), cloudFile);
+ }
+
+ return map;
}
}
diff --git
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
index dcc1b93c7e..e4ad05e629 100644
---
a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
+++
b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
@@ -58,9 +58,12 @@ public class CloudFileUtil {
continue;
}
- CloudFile path = CloudFile.of(file.getRelativePath());
+ CloudFile path = CloudFile.of(file.getRelativePath(),
ioManager.getSize(file));
if (!cloudFiles.contains(path)) {
- // Delete local files that do not exist in cloud storage (the
ground truth for valid files)
+ /*
+ * Delete local files that do not exist in cloud storage (the
ground truth for valid files), or files
+ * that has not been downloaded completely.
+ */
logDeleteFile(file);
localFilesIter.remove();
ioManager.delete(file);
@@ -82,7 +85,8 @@ public class CloudFileUtil {
}
private static void logDeleteFile(FileReference fileReference) {
- LOGGER.info("Deleting {} from the local cache as {} doesn't exist in
the cloud", fileReference,
- fileReference.getRelativePath());
+ LOGGER.info(
+ "Deleting {} from the local cache as {} either doesn't exist
in the cloud or it wasn't downloaded completely",
+ fileReference, fileReference.getRelativePath());
}
}
