>From Ritik Raj <[email protected]>:
Ritik Raj has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18388 )
Change subject: [ASTERIXDB-3439][STO] Exposing cloud read metrics
......................................................................
[ASTERIXDB-3439][STO] Exposing cloud read metrics
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
This patch adds following three metrics related to cloud reads
1. cloudRequest - Gets the count of cloud request to object storage
2. cloudPageRead - The count of pages read from the cloud
3. cloudPagePersist - The count of fetched page is persisted in the disk.
Change-Id: I5d547e4267ec93c97f7b5398ce925fbb73262010
---
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
M
hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadStats.java
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/SweepBufferCacheReadContext.java
M
hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
M
hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/buffercache/context/DefaultCloudReadContext.java
M
hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudMegaPageReadContext.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
M
hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/job/profiling/StatsCollector.java
M
hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/JobResultCallback.java
M
hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/DefaultColumnReadContext.java
M
asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ResultMetadata.java
M
hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/IThreadStats.java
M
hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java
M
hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/CompressedBufferedFileHandle.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java
M
hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/context/read/AbstractBufferCacheReadContext.java
M
hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResponseMetrics.java
M
hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
M
hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/context/IBufferCacheReadContext.java
27 files changed, 314 insertions(+), 33 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/88/18388/1
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
index b624c9b..a13b7ae 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
@@ -140,6 +140,9 @@
private long compileTime;
private double bufferCacheHitRatio;
private long bufferCachePageReadCount;
+ private long cloudRequestsCount;
+ private long cloudPagesReadCount;
+ private long cloudPagesPersistedCount;
public long getCount() {
return count;
@@ -222,6 +225,30 @@
public long getBufferCachePageReadCount() {
return bufferCachePageReadCount;
}
+
+ public void setCloudRequestsCount(long cloudRequestsCount) {
+ this.cloudRequestsCount = cloudRequestsCount;
+ }
+
+ public long getCloudRequestsCount() {
+ return cloudRequestsCount;
+ }
+
+ public void setCloudPagesReadCount(long cloudPagesReadCount) {
+ this.cloudPagesReadCount = cloudPagesReadCount;
+ }
+
+ public long getCloudPagesReadCount() {
+ return cloudPagesReadCount;
+ }
+
+ public void setCloudPagesPersistedCount(long cloudPagesPersistedCount)
{
+ this.cloudPagesPersistedCount = cloudPagesPersistedCount;
+ }
+
+ public long getCloudPagesPersistedCount() {
+ return cloudPagesPersistedCount;
+ }
}
class Profile implements Serializable {
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ResultMetadata.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ResultMetadata.java
index af06817..159db9c 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ResultMetadata.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ResultMetadata.java
@@ -40,6 +40,9 @@
private transient List<Object> outputTypes;
private long queueWaitTimeInNanos;
private long bufferCachePageReadCount;
+ private long cloudRequestsCount;
+ private long cloudPagesReadCount;
+ private long cloudPagesPersistedCount;
public ResultMetadata(SessionConfig.OutputFormat format) {
this.format = format;
@@ -77,6 +80,30 @@
return bufferCachePageReadCount;
}
+ public void setCloudRequestsCount(long cloudRequestsCount) {
+ this.cloudRequestsCount = cloudRequestsCount;
+ }
+
+ public long getCloudRequestsCount() {
+ return cloudRequestsCount;
+ }
+
+ public void setCloudPagesReadCount(long cloudPagesReadCount) {
+ this.cloudPagesReadCount = cloudPagesReadCount;
+ }
+
+ public long getCloudPagesReadCount() {
+ return cloudPagesReadCount;
+ }
+
+ public void setCloudPagesPersistedCount(long cloudPagesPersistedCount) {
+ this.cloudPagesPersistedCount = cloudPagesPersistedCount;
+ }
+
+ public long getCloudPagesPersistedCount() {
+ return cloudPagesPersistedCount;
+ }
+
public void setWarnings(Set<Warning> warnings) {
this.warnings = warnings;
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
index 65d4734..42b0ea7 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
@@ -209,6 +209,9 @@
stats.setQueueWaitTime(responseStats.getQueueWaitTime());
stats.setBufferCacheHitRatio(responseStats.getBufferCacheHitRatio());
stats.setBufferCachePageReadCount(responseStats.getBufferCachePageReadCount());
+ stats.setCloudRequestsCount(responseStats.getCloudRequestsCount());
+ stats.setCloudPagesReadCount(responseStats.getCloudPagesReadCount());
+
stats.setCloudPagesPersistedCount(responseStats.getCloudPagesPersistedCount());
}
private static void
updatePropertiesFromCC(IStatementExecutor.StatementProperties
statementProperties,
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
index c2e8dca..f1be007 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
@@ -103,7 +103,9 @@
ResponseMetrics metrics = ResponseMetrics.of(System.nanoTime()
- elapsedStart,
metadata.getJobDuration(), stats.getCount(),
stats.getSize(), metadata.getProcessedObjects(), 0,
metadata.getTotalWarningsCount(),
stats.getCompileTime(), stats.getQueueWaitTime(),
- stats.getBufferCacheHitRatio(),
stats.getBufferCachePageReadCount());
+ stats.getBufferCacheHitRatio(),
stats.getBufferCachePageReadCount(),
+ stats.getCloudRequestsCount(),
stats.getCloudPagesReadCount(),
+ stats.getCloudPagesPersistedCount());
printer.addFooterPrinter(new MetricsPrinter(metrics,
HttpUtil.getPreferredCharset(request)));
if (metadata.getJobProfile() != null) {
printer.addFooterPrinter(new
ProfilePrinter(metadata.getJobProfile()));
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index 4798f9e..77875c6 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -369,7 +369,8 @@
final ResponseMetrics metrics = ResponseMetrics.of(System.nanoTime() -
elapsedStart, executionState.duration(),
stats.getCount(), stats.getSize(),
stats.getProcessedObjects(), errorCount,
stats.getTotalWarningsCount(), stats.getCompileTime(),
stats.getQueueWaitTime(),
- stats.getBufferCacheHitRatio(),
stats.getBufferCachePageReadCount());
+ stats.getBufferCacheHitRatio(),
stats.getBufferCachePageReadCount(), stats.getCloudRequestsCount(),
+ stats.getCloudPagesReadCount(),
stats.getCloudPagesPersistedCount());
responsePrinter.addFooterPrinter(new MetricsPrinter(metrics,
resultCharset));
if (isPrintingProfile(stats)) {
responsePrinter.addFooterPrinter(new
ProfilePrinter(stats.getJobProfile()));
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/JobResultCallback.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/JobResultCallback.java
index c419d0d..2018fd5 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/JobResultCallback.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/JobResultCallback.java
@@ -74,6 +74,9 @@
long aggregateTotalWarningsCount = 0;
long pagesRead = 0;
long nonPagedReads = 0;
+ long cloudRequestsCount = 0;
+ long cloudPagesRead = 0;
+ long cloudPagesPersisted = 0;
Set<Warning> AggregateWarnings = new HashSet<>();
IJobManager jobManager =
((ClusterControllerService)
appCtx.getServiceContext().getControllerService()).getJobManager();
@@ -88,6 +91,9 @@
processedObjects +=
tp.getStatsCollector().getAggregatedStats().getInputTupleCounter().get();
pagesRead +=
tp.getStatsCollector().getAggregatedStats().getPageReads().get();
nonPagedReads +=
tp.getStatsCollector().getAggregatedStats().coldReadCounter().get();
+ cloudRequestsCount +=
tp.getStatsCollector().getAggregatedStats().cloudRequestCounter().get();
+ cloudPagesRead +=
tp.getStatsCollector().getAggregatedStats().cloudReadPageCounter().get();
+ cloudPagesPersisted +=
tp.getStatsCollector().getAggregatedStats().cloudPersistPageCounter().get();
aggregateTotalWarningsCount += tp.getTotalWarningsCount();
Set<Warning> taskWarnings = tp.getWarnings();
if (AggregateWarnings.size() < maxWarnings &&
!taskWarnings.isEmpty()) {
@@ -103,6 +109,9 @@
metadata.setProcessedObjects(processedObjects);
metadata.setBufferCacheHitRatio(pagesRead > 0 ? (pagesRead -
nonPagedReads) / (double) pagesRead : Double.NaN);
metadata.setBufferCachePageReadCount(pagesRead);
+ metadata.setCloudRequestsCount(cloudRequestsCount);
+ metadata.setCloudPagesReadCount(cloudPagesRead);
+ metadata.setCloudPagesPersistedCount(cloudPagesPersisted);
metadata.setWarnings(AggregateWarnings);
metadata.setTotalWarningsCount(aggregateTotalWarningsCount);
if (run != null && run.getFlags() != null &&
run.getFlags().contains(JobFlag.PROFILE_RUNTIME)) {
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResponseMetrics.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResponseMetrics.java
index 8db483c..5681cd9 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResponseMetrics.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResponseMetrics.java
@@ -32,13 +32,17 @@
private long queueWaitTime;
private double bufferCacheHitRatio;
private long bufferCachePageReadCount;
+ private long cloudRequestsCount;
+ private long cloudPagesReadCount;
+ private long cloudPagesPersistedCount;
private ResponseMetrics() {
}
public static ResponseMetrics of(long elapsedTime, long executionTime,
long resultCount, long resultSize,
long processedObjects, long errorCount, long warnCount, long
compileTime, long queueWaitTime,
- double bufferCacheHitRatio, long bufferCachePageReadCount) {
+ double bufferCacheHitRatio, long bufferCachePageReadCount, long
cloudRequestsCount,
+ long cloudPagesReadCount, long cloudPagesPersistedCount) {
ResponseMetrics metrics = new ResponseMetrics();
metrics.elapsedTime = elapsedTime;
metrics.executionTime = executionTime;
@@ -51,6 +55,9 @@
metrics.queueWaitTime = queueWaitTime;
metrics.bufferCacheHitRatio = bufferCacheHitRatio;
metrics.bufferCachePageReadCount = bufferCachePageReadCount;
+ metrics.cloudRequestsCount = cloudRequestsCount;
+ metrics.cloudPagesReadCount = cloudPagesReadCount;
+ metrics.cloudPagesPersistedCount = cloudPagesPersistedCount;
return metrics;
}
@@ -97,4 +104,16 @@
public long getBufferCachePageReadCount() {
return bufferCachePageReadCount;
}
+
+ public long getCloudRequestsCount() {
+ return cloudRequestsCount;
+ }
+
+ public long getCloudPagesReadCount() {
+ return cloudPagesReadCount;
+ }
+
+ public long getCloudPagesPersistedCount() {
+ return cloudPagesPersistedCount;
+ }
}
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
index 76ad2a4..cfbe281 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
@@ -40,7 +40,10 @@
PROCESSED_OBJECTS_COUNT("processedObjects"),
WARNING_COUNT("warningCount"),
BUFFERCACHE_HIT_RATIO("bufferCacheHitRatio"),
- BUFFERCACHE_PAGEREAD_COUNT("bufferCachePageReadCount");
+ BUFFERCACHE_PAGEREAD_COUNT("bufferCachePageReadCount"),
+ CLOUD_REQUESTS_COUNT("cloudRequestsCount"),
+ CLOUD_PAGES_READ_COUNT("cloudPagesReadCount"),
+ CLOUD_PAGES_PERSISTED_COUNT("cloudPagesPersistedCount");
private final String str;
@@ -87,6 +90,7 @@
final boolean hasErrors = metrics.getErrorCount() > 0;
final boolean hasWarnings = metrics.getWarnCount() > 0;
final boolean usedCache =
!(Double.isNaN(metrics.getBufferCacheHitRatio()));
+ final boolean madeCloudRequests = metrics.getCloudRequestsCount() > 0;
ResultUtil.printField(pw, Metrics.PROCESSED_OBJECTS_COUNT.str(),
metrics.getProcessedObjects(),
usedCache || hasWarnings || hasErrors);
pw.print("\n");
@@ -100,6 +104,18 @@
hasWarnings || hasErrors);
pw.print("\n");
}
+ if (madeCloudRequests) {
+ pw.print("\t");
+ ResultUtil.printField(pw, Metrics.CLOUD_REQUESTS_COUNT.str(),
metrics.getCloudRequestsCount(), true);
+ pw.print("\n");
+ pw.print("\t");
+ ResultUtil.printField(pw, Metrics.CLOUD_PAGES_READ_COUNT.str(),
metrics.getCloudPagesReadCount(), true);
+ pw.print("\n");
+ pw.print("\t");
+ ResultUtil.printField(pw,
Metrics.CLOUD_PAGES_PERSISTED_COUNT.str(),
metrics.getCloudPagesPersistedCount(),
+ true);
+ pw.print("\n");
+ }
if (hasWarnings) {
pw.print("\t");
ResultUtil.printField(pw, Metrics.WARNING_COUNT.str(),
metrics.getWarnCount(), hasErrors);
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 340cd57..ae718ac 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -5334,6 +5334,9 @@
stats.setQueueWaitTime(resultMetadata.getQueueWaitTimeInNanos());
stats.setBufferCacheHitRatio(resultMetadata.getBufferCacheHitRatio());
stats.setBufferCachePageReadCount(resultMetadata.getBufferCachePageReadCount());
+ stats.setCloudRequestsCount(resultMetadata.getCloudRequestsCount());
+ stats.setCloudPagesReadCount(resultMetadata.getCloudPagesReadCount());
+
stats.setCloudPagesPersistedCount(resultMetadata.getCloudPagesPersistedCount());
if (jobFlags.contains(JobFlag.PROFILE_RUNTIME)) {
stats.setJobProfile(resultMetadata.getJobProfile());
apiFramework.generateOptimizedLogicalPlanWithProfile(resultMetadata.getJobProfile());
diff --git
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java
index 7770c4f..38eb667 100644
---
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java
+++
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/IOperatorStats.java
@@ -57,6 +57,21 @@
ICounter coldReadCounter();
/**
+ * @return A counter used to track the number of request to fetch pages
from object store
+ */
+ ICounter cloudRequestCounter();
+
+ /**
+ * @return A counter used to track the number of pages read from the cloud.
+ */
+ ICounter cloudReadPageCounter();
+
+ /**
+ * @return A counter used to track the number of pages, fetched form cloud
gets persisted to disk
+ */
+ ICounter cloudPersistPageCounter();
+
+ /**
* @return A counter used to set the average tuple size outputted by an
operator
*/
diff --git
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java
index d427d14..cd44130 100644
---
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java
+++
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/NoOpOperatorStats.java
@@ -95,6 +95,21 @@
}
@Override
+ public ICounter cloudRequestCounter() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
+ public ICounter cloudReadPageCounter() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
+ public ICounter cloudPersistPageCounter() {
+ return NOOP_COUNTER;
+ }
+
+ @Override
public ICounter getAverageTupleSz() {
return NOOP_COUNTER;
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java
index 412b788..48c14d3 100644
---
a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java
+++
b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/job/profiling/OperatorStats.java
@@ -36,6 +36,9 @@
public final ICounter timeCounter;
public final ICounter pageReads;
public final ICounter coldReadCounter;
+ public final ICounter cloudRequestCounter;
+ public final ICounter cloudReadPageCounter;
+ public final ICounter cloudPersistPageCounter;
public final ICounter avgTupleSz;
public final ICounter minTupleSz;
public final ICounter maxTupleSz;
@@ -57,6 +60,9 @@
timeCounter = new Counter("timeCounter");
pageReads = new Counter("diskIoCounter");
coldReadCounter = new Counter("coldReadCounter");
+ cloudRequestCounter = new Counter("cloudRequestCounter");
+ cloudReadPageCounter = new Counter("cloudReadPageCounter");
+ cloudPersistPageCounter = new Counter("cloudPersistPageCounter");
avgTupleSz = new Counter("avgTupleSz");
minTupleSz = new Counter("minTupleSz");
maxTupleSz = new Counter("maxTupleSz");
@@ -94,6 +100,21 @@
}
@Override
+ public ICounter cloudRequestCounter() {
+ return cloudRequestCounter;
+ }
+
+ @Override
+ public ICounter cloudReadPageCounter() {
+ return cloudReadPageCounter;
+ }
+
+ @Override
+ public ICounter cloudPersistPageCounter() {
+ return cloudPersistPageCounter;
+ }
+
+ @Override
public ICounter getAverageTupleSz() {
return avgTupleSz;
}
@@ -171,6 +192,9 @@
output.writeLong(timeCounter.get());
output.writeLong(pageReads.get());
output.writeLong(coldReadCounter.get());
+ output.writeLong(cloudRequestCounter.get());
+ output.writeLong(cloudReadPageCounter.get());
+ output.writeLong(cloudPersistPageCounter.get());
output.writeLong(avgTupleSz.get());
output.writeLong(minTupleSz.get());
output.writeLong(maxTupleSz.get());
@@ -187,6 +211,9 @@
timeCounter.set(input.readLong());
pageReads.set(input.readLong());
coldReadCounter.set(input.readLong());
+ cloudRequestCounter.set(input.readLong());
+ cloudReadPageCounter.set(input.readLong());
+ cloudPersistPageCounter.set(input.readLong());
avgTupleSz.set(input.readLong());
minTupleSz.set(input.readLong());
maxTupleSz.set(input.readLong());
@@ -218,11 +245,14 @@
public String toString() {
return "{ " + "\"operatorName\": \"" + operatorName + "\", " +
"\"id\": \"" + operatorId + "\", " + "\""
+ tupleCounter.getName() + "\": " + tupleCounter.get() + ",
\"" + timeCounter.getName() + "\": "
- + timeCounter.get() + ", \"" + coldReadCounter.getName() +
"\": " + coldReadCounter.get()
- + avgTupleSz.getName() + "\": " + avgTupleSz.get() + ", \"" +
minTupleSz.getName() + "\": "
- + minTupleSz.get() + ", \"" + minTupleSz.getName() + "\": " +
timeCounter.get() + ", \""
- + inputTupleCounter.getName() + "\": " + bytesRead.get() + ",
\"" + bytesRead.getName() + "\": "
- + bytesWritten.get() + ", \"" + bytesWritten.getName() + "\":
" + inputTupleCounter.get() + ", \""
- + level.getName() + "\": " + level.get() + ", \"indexStats\":
\"" + indexesStats + "\" }";
+ + timeCounter.get() + ", \"" + coldReadCounter.getName() +
"\": " + coldReadCounter.get() + ", \""
+ + cloudRequestCounter.getName() + "\": " +
cloudRequestCounter.get() + ", \""
+ + cloudReadPageCounter.getName() + "\": " +
cloudReadPageCounter.get() + ", \""
+ + cloudPersistPageCounter.getName() + "\": " +
cloudPersistPageCounter.get() + avgTupleSz.getName()
+ + "\": " + avgTupleSz.get() + ", \"" + minTupleSz.getName() +
"\": " + minTupleSz.get() + ", \""
+ + minTupleSz.getName() + "\": " + timeCounter.get() + ", \"" +
inputTupleCounter.getName() + "\": "
+ + bytesRead.get() + ", \"" + bytesRead.getName() + "\": " +
bytesWritten.get() + ", \""
+ + bytesWritten.getName() + "\": " + inputTupleCounter.get() +
", \"" + level.getName() + "\": "
+ + level.get() + ", \"indexStats\": \"" + indexesStats + "\" }";
}
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/buffercache/context/DefaultCloudReadContext.java
b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/buffercache/context/DefaultCloudReadContext.java
index 9c24ce1..4ec2d6f 100644
---
a/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/buffercache/context/DefaultCloudReadContext.java
+++
b/hyracks-fullstack/hyracks/hyracks-cloud/src/main/java/org/apache/hyracks/cloud/buffercache/context/DefaultCloudReadContext.java
@@ -34,6 +34,7 @@
import
org.apache.hyracks.storage.common.buffercache.context.IBufferCacheReadContext;
import org.apache.hyracks.storage.common.disk.IPhysicalDrive;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.util.IThreadStats;
import org.apache.hyracks.util.annotations.ThreadSafe;
/**
@@ -73,7 +74,7 @@
@Override
public ByteBuffer processHeader(IOManager ioManager, BufferedFileHandle
fileHandle, BufferCacheHeaderHelper header,
- CachedPage cPage) throws HyracksDataException {
+ CachedPage cPage, IThreadStats threadStats) throws
HyracksDataException {
return readAndPersistPage(ioManager, fileHandle, header, cPage,
drive.hasSpace());
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/job/profiling/StatsCollector.java
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/job/profiling/StatsCollector.java
index 7cfbda0..89cf8f0 100644
---
a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/job/profiling/StatsCollector.java
+++
b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/job/profiling/StatsCollector.java
@@ -69,6 +69,9 @@
aggregatedStats.getTimeCounter().update(stats.getTimeCounter().get());
aggregatedStats.getPageReads().update(stats.getPageReads().get());
aggregatedStats.coldReadCounter().update(stats.coldReadCounter().get());
+
aggregatedStats.cloudRequestCounter().update(stats.cloudRequestCounter().get());
+
aggregatedStats.cloudReadPageCounter().update(stats.cloudReadPageCounter().get());
+
aggregatedStats.cloudPersistPageCounter().update(stats.cloudPersistPageCounter().get());
}
return aggregatedStats;
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
index 63ff2ed..bad399e 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java
@@ -344,6 +344,9 @@
}
stats.getPageReads().update(ctx.getThreadStats().getPinnedPagesCount());
stats.coldReadCounter().update(ctx.getThreadStats().getColdReadCount());
+
stats.cloudRequestCounter().update(ctx.getThreadStats().getCloudRequestCount());
+
stats.cloudReadPageCounter().update(ctx.getThreadStats().getCloudPageReadCount());
+
stats.cloudPersistPageCounter().update(ctx.getThreadStats().getCloudPagePersistCount());
} catch (Throwable th) { // NOSONAR Must ensure writer.fail is
called.
// subsequently, the failure will be thrown
failure = th;
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java
index fef150a..7919f74 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudColumnReadContext.java
@@ -42,6 +42,7 @@
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.disk.IPhysicalDrive;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.util.IThreadStats;
import org.apache.hyracks.util.annotations.NotThreadSafe;
@NotThreadSafe
@@ -104,7 +105,7 @@
@Override
public ByteBuffer processHeader(IOManager ioManager, BufferedFileHandle
fileHandle, BufferCacheHeaderHelper header,
- CachedPage cPage) throws HyracksDataException {
+ CachedPage cPage, IThreadStats threadStats) throws
HyracksDataException {
// Page zero will be persisted (always) if free space permits
return readAndPersistPage(ioManager, fileHandle, header, cPage,
drive.hasSpace());
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudMegaPageReadContext.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudMegaPageReadContext.java
index ed6e83f..b606a91 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudMegaPageReadContext.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/CloudMegaPageReadContext.java
@@ -42,6 +42,7 @@
import
org.apache.hyracks.storage.common.buffercache.context.IBufferCacheReadContext;
import org.apache.hyracks.storage.common.disk.IPhysicalDrive;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.util.IThreadStats;
import org.apache.hyracks.util.annotations.NotThreadSafe;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -112,7 +113,7 @@
@Override
public ByteBuffer processHeader(IOManager ioManager, BufferedFileHandle
fileHandle, BufferCacheHeaderHelper header,
- CachedPage cPage) throws HyracksDataException {
+ CachedPage cPage, IThreadStats threadStats) throws
HyracksDataException {
boolean empty = BufferCacheCloudReadContextUtil.isEmpty(header);
int pageId = BufferedFileHandle.getPageId(cPage.getDiskPageId());
boolean cloudOnly = columnRanges.isCloudOnly(pageId);
@@ -131,7 +132,7 @@
* 'cloudOnly' is true.
*/
boolean persist = empty && !cloudOnly && !evictable && operation
!= MERGE && drive.isUnpressured();
- readFromStream(ioManager, fileHandle, header, cPage, persist);
+ readFromStream(ioManager, fileHandle, header, cPage, threadStats,
persist);
} else {
/*
* Here we can find a page that is planned for eviction, but it
has not being evicted yet
@@ -142,7 +143,7 @@
}
// Finally process the header
- return DEFAULT.processHeader(ioManager, fileHandle, header, cPage);
+ return DEFAULT.processHeader(ioManager, fileHandle, header, cPage,
threadStats);
}
void unpinAll(IBufferCache bufferCache) throws HyracksDataException {
@@ -168,8 +169,8 @@
}
private void readFromStream(IOManager ioManager, BufferedFileHandle
fileHandle, BufferCacheHeaderHelper header,
- CachedPage cPage, boolean persist) throws HyracksDataException {
- InputStream stream = getOrCreateStream(ioManager, fileHandle, cPage);
+ CachedPage cPage, IThreadStats threadStats, boolean persist)
throws HyracksDataException {
+ InputStream stream = getOrCreateStream(ioManager, fileHandle, cPage,
threadStats);
ByteBuffer buffer = header.getBuffer();
buffer.position(0);
@@ -203,14 +204,20 @@
long offset = cPage.getCompressedPageOffset();
ICloudIOManager cloudIOManager = (ICloudIOManager) ioManager;
BufferCacheCloudReadContextUtil.persist(cloudIOManager,
fileHandle.getFileHandle(), buffer, offset);
+ if (threadStats != null) {
+ threadStats.cloudPagePersist();
+ }
}
streamOffset += cPage.getCompressedPageSize();
remainingStreamBytes -= cPage.getCompressedPageSize();
+ if (threadStats != null) {
+ threadStats.cloudPageRead();
+ }
}
- private InputStream getOrCreateStream(IOManager ioManager,
BufferedFileHandle fileHandle, CachedPage cPage)
- throws HyracksDataException {
+ private InputStream getOrCreateStream(IOManager ioManager,
BufferedFileHandle fileHandle, CachedPage cPage,
+ IThreadStats threadStats) throws HyracksDataException {
if (gapStream != null) {
return gapStream;
}
@@ -229,6 +236,9 @@
ICloudIOManager cloudIOManager = (ICloudIOManager) ioManager;
gapStream = cloudIOManager.cloudRead(fileHandle.getFileHandle(),
offset, length);
+ if (threadStats != null) {
+ threadStats.cloudRequest();
+ }
return gapStream;
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/DefaultColumnReadContext.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/DefaultColumnReadContext.java
index 0e6f858..11275a4 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/DefaultColumnReadContext.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/buffercache/read/DefaultColumnReadContext.java
@@ -31,6 +31,7 @@
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.util.IThreadStats;
public final class DefaultColumnReadContext implements IColumnReadContext {
public static final IColumnReadContext INSTANCE = new
DefaultColumnReadContext();
@@ -60,8 +61,8 @@
@Override
public ByteBuffer processHeader(IOManager ioManager, BufferedFileHandle
fileHandle, BufferCacheHeaderHelper header,
- CachedPage cPage) throws HyracksDataException {
- return DEFAULT.processHeader(ioManager, fileHandle, header, cPage);
+ CachedPage cPage, IThreadStats threadStats) throws
HyracksDataException {
+ return DEFAULT.processHeader(ioManager, fileHandle, header, cPage,
threadStats);
}
@Override
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/SweepBufferCacheReadContext.java
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/SweepBufferCacheReadContext.java
index 7b51d55..ef61d76 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/SweepBufferCacheReadContext.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree-column/src/main/java/org/apache/hyracks/storage/am/lsm/btree/column/cloud/sweep/SweepBufferCacheReadContext.java
@@ -29,6 +29,7 @@
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import
org.apache.hyracks.storage.common.buffercache.context.IBufferCacheReadContext;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.util.IThreadStats;
import org.apache.hyracks.util.annotations.ThreadSafe;
@ThreadSafe
@@ -61,7 +62,7 @@
@Override
public ByteBuffer processHeader(IOManager ioManager, BufferedFileHandle
fileHandle, BufferCacheHeaderHelper header,
- CachedPage cPage) throws HyracksDataException {
+ CachedPage cPage, IThreadStats threadStats) throws
HyracksDataException {
// Will not persist as the disk is pressured
return readAndPersistPage(ioManager, fileHandle, header, cPage, false);
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java
index a0ad045..c211573 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/AbstractBufferedFileIOManager.java
@@ -31,6 +31,7 @@
import
org.apache.hyracks.storage.common.buffercache.context.IBufferCacheWriteContext;
import
org.apache.hyracks.storage.common.compression.file.CompressedFileReference;
import
org.apache.hyracks.storage.common.compression.file.ICompressedPageWriter;
+import org.apache.hyracks.util.IThreadStats;
import org.apache.hyracks.util.annotations.NotThreadSafe;
/**
@@ -70,7 +71,8 @@
* @param cPage CachedPage in {@link BufferCache}
* @param context read context
*/
- public abstract void read(CachedPage cPage, IBufferCacheReadContext
context) throws HyracksDataException;
+ public abstract void read(CachedPage cPage, IBufferCacheReadContext
context, IThreadStats threadStats)
+ throws HyracksDataException;
/**
* Write the CachedPage into disk
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
index d2a6fc0..7f0cac3 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
@@ -567,14 +567,14 @@
private void read(CachedPage cPage, IBufferCacheReadContext context)
throws HyracksDataException {
BufferedFileHandle fInfo = getFileHandle(cPage);
cPage.buffer.clear();
+ final IThreadStats threadStats =
statsSubscribers.get(Thread.currentThread());
try {
- fInfo.read(cPage, context);
+ fInfo.read(cPage, context, threadStats);
} catch (Throwable e) {
LOGGER.error("Error while reading a page {} in file {}", cPage,
fInfo);
throw e;
}
- final IThreadStats threadStats =
statsSubscribers.get(Thread.currentThread());
if (threadStats != null && context.incrementStats()) {
threadStats.coldRead();
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/context/IBufferCacheReadContext.java
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/context/IBufferCacheReadContext.java
index 3c2bbcd..c2f4921 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/context/IBufferCacheReadContext.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/context/IBufferCacheReadContext.java
@@ -27,6 +27,7 @@
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.util.IThreadStats;
/**
* Provide a context to {@link IBufferCache} pin/unpin operations as well as
processing the header of the first
@@ -68,5 +69,5 @@
* @return the byte buffer of the header after processing it
*/
ByteBuffer processHeader(IOManager ioManager, BufferedFileHandle
fileHandle, BufferCacheHeaderHelper header,
- CachedPage cPage) throws HyracksDataException;
+ CachedPage cPage, IThreadStats threadStats) throws
HyracksDataException;
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/context/read/AbstractBufferCacheReadContext.java
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/context/read/AbstractBufferCacheReadContext.java
index 42abd2e..927b437 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/context/read/AbstractBufferCacheReadContext.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/context/read/AbstractBufferCacheReadContext.java
@@ -30,6 +30,7 @@
import org.apache.hyracks.storage.common.buffercache.ICachedPage;
import
org.apache.hyracks.storage.common.buffercache.context.IBufferCacheReadContext;
import org.apache.hyracks.storage.common.file.BufferedFileHandle;
+import org.apache.hyracks.util.IThreadStats;
abstract class AbstractBufferCacheReadContext implements
IBufferCacheReadContext {
@Override
@@ -44,7 +45,7 @@
@Override
public final ByteBuffer processHeader(IOManager ioManager,
BufferedFileHandle fileHandle,
- BufferCacheHeaderHelper header, CachedPage cPage) {
+ BufferCacheHeaderHelper header, CachedPage cPage, IThreadStats
threadStats) {
ByteBuffer buf = header.getBuffer();
cPage.setFrameSizeMultiplier(buf.getInt(FRAME_MULTIPLIER_OFF));
cPage.setExtraBlockPageId(buf.getInt(EXTRA_BLOCK_PAGE_ID_OFF));
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java
index ff2bd83..9704cdb 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/BufferedFileHandle.java
@@ -38,6 +38,7 @@
import
org.apache.hyracks.storage.common.compression.file.CompressedFileReference;
import
org.apache.hyracks.storage.common.compression.file.ICompressedPageWriter;
import org.apache.hyracks.storage.common.compression.file.NoOpLAFWriter;
+import org.apache.hyracks.util.IThreadStats;
public class BufferedFileHandle extends AbstractBufferedFileIOManager {
private final int fileId;
@@ -71,7 +72,8 @@
}
@Override
- public void read(CachedPage cPage, IBufferCacheReadContext context) throws
HyracksDataException {
+ public void read(CachedPage cPage, IBufferCacheReadContext context,
IThreadStats threadStats)
+ throws HyracksDataException {
final BufferCacheHeaderHelper header = checkoutHeaderHelper();
try {
setPageInfo(cPage);
@@ -83,7 +85,7 @@
return;
}
- final ByteBuffer buf = context.processHeader(ioManager, this,
header, cPage);
+ final ByteBuffer buf = context.processHeader(ioManager, this,
header, cPage, threadStats);
cPage.getBuffer().put(buf);
} finally {
returnHeaderHelper(header);
diff --git
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/CompressedBufferedFileHandle.java
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/CompressedBufferedFileHandle.java
index cd882b5..6ad4d27 100644
---
a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/CompressedBufferedFileHandle.java
+++
b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/file/CompressedBufferedFileHandle.java
@@ -35,6 +35,7 @@
import
org.apache.hyracks.storage.common.compression.file.CompressedFileManager;
import
org.apache.hyracks.storage.common.compression.file.CompressedFileReference;
import
org.apache.hyracks.storage.common.compression.file.ICompressedPageWriter;
+import org.apache.hyracks.util.IThreadStats;
public class CompressedBufferedFileHandle extends BufferedFileHandle {
private final FileReference lafFileRef;
@@ -48,7 +49,8 @@
}
@Override
- public void read(CachedPage cPage, IBufferCacheReadContext context) throws
HyracksDataException {
+ public void read(CachedPage cPage, IBufferCacheReadContext context,
IThreadStats threadStats)
+ throws HyracksDataException {
final BufferCacheHeaderHelper header = checkoutHeaderHelper();
try {
compressedFileManager.setCompressedPageInfo(cPage);
@@ -59,7 +61,7 @@
return;
}
- final ByteBuffer cBuffer = context.processHeader(ioManager, this,
header, cPage);
+ final ByteBuffer cBuffer = context.processHeader(ioManager, this,
header, cPage, threadStats);
final ByteBuffer uBuffer = cPage.getBuffer();
fixBufferPointers(uBuffer, 0);
if (cPage.getCompressedPageSize() <
bufferCache.getPageSizeWithHeader()) {
diff --git
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/IThreadStats.java
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/IThreadStats.java
index e782a69..58ec724 100644
---
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/IThreadStats.java
+++
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/IThreadStats.java
@@ -43,4 +43,37 @@
* @return the cold read count
*/
long getColdReadCount();
+
+ /**
+ * Indicates that this thread made a cloud request to object storage
+ */
+ void cloudRequest();
+
+ /**
+ * Gets the count of cloud request to object storage
+ *
+ * @return the cloud request count
+ */
+ long getCloudRequestCount();
+
+ /**
+ * Indicates a page is read from the cloud
+ */
+ void cloudPageRead();
+
+ /**
+ * @return the count of pages read from the cloud
+ */
+ long getCloudPageReadCount();
+
+ /**
+ * Indicates the page is persistent in the disk,
+ * after fetching from cloud.
+ */
+ void cloudPagePersist();
+
+ /**
+ * @return the count of fetched page is persisted in the disk.
+ */
+ long getCloudPagePersistCount();
}
diff --git
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadStats.java
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadStats.java
index 9af3fb1..ada90cc 100644
---
a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadStats.java
+++
b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadStats.java
@@ -25,8 +25,11 @@
@ThreadSafe
public class ThreadStats implements IThreadStats {
- private AtomicLong pinnedPagesCount = new AtomicLong();
- private AtomicLong coldReadCount = new AtomicLong();
+ private final AtomicLong pinnedPagesCount = new AtomicLong();
+ private final AtomicLong coldReadCount = new AtomicLong();
+ private final AtomicLong cloudRequestCount = new AtomicLong();
+ private final AtomicLong cloudReadPageCount = new AtomicLong();
+ private final AtomicLong cloudPersistPageCount = new AtomicLong();
@Override
public void pagePinned() {
@@ -47,4 +50,34 @@
public void coldRead() {
coldReadCount.incrementAndGet();
}
+
+ @Override
+ public void cloudRequest() {
+ cloudRequestCount.incrementAndGet();
+ }
+
+ @Override
+ public long getCloudRequestCount() {
+ return cloudRequestCount.get();
+ }
+
+ @Override
+ public void cloudPageRead() {
+ cloudReadPageCount.incrementAndGet();
+ }
+
+ @Override
+ public long getCloudPageReadCount() {
+ return cloudReadPageCount.get();
+ }
+
+ @Override
+ public void cloudPagePersist() {
+ cloudPersistPageCount.incrementAndGet();
+ }
+
+ @Override
+ public long getCloudPagePersistCount() {
+ return cloudPersistPageCount.get();
+ }
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18388
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I5d547e4267ec93c97f7b5398ce925fbb73262010
Gerrit-Change-Number: 18388
Gerrit-PatchSet: 1
Gerrit-Owner: Ritik Raj <[email protected]>
Gerrit-MessageType: newchange
