This is an automated email from the ASF dual-hosted git repository.
hellostephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2f65127ccb4 [spill](regression test) check spill temporary files are
deleted after all cases are done (#37248)
2f65127ccb4 is described below
commit 2f65127ccb4f3a0495d83e5956788b8ff7a161c1
Author: TengJianPing <[email protected]>
AuthorDate: Thu Jul 4 14:56:15 2024 +0800
[spill](regression test) check spill temporary files are deleted after all
cases are done (#37248)
## Proposed changes
Issue Number: close #xxx
<!--Describe your changes.-->
---
be/src/vec/spill/spill_stream_manager.cpp | 2 +-
.../check_before_quit/check_before_quit.groovy | 77 ++++++++++++++++++++++
2 files changed, 78 insertions(+), 1 deletion(-)
diff --git a/be/src/vec/spill/spill_stream_manager.cpp
b/be/src/vec/spill/spill_stream_manager.cpp
index 77ecea8d424..61e96559d23 100644
--- a/be/src/vec/spill/spill_stream_manager.cpp
+++ b/be/src/vec/spill/spill_stream_manager.cpp
@@ -290,7 +290,7 @@ SpillDataDir::SpillDataDir(std::string path, int64_t
capacity_bytes,
_disk_capacity_bytes(capacity_bytes),
_storage_medium(storage_medium) {
spill_data_dir_metric_entity =
DorisMetrics::instance()->metric_registry()->register_entity(
- std::string("spill_data_dir.") + path, {{"path", path}});
+ std::string("spill_data_dir.") + _path, {{"path", _path + "/" +
SPILL_DIR_PREFIX}});
INT_GAUGE_METRIC_REGISTER(spill_data_dir_metric_entity,
spill_disk_capacity);
INT_GAUGE_METRIC_REGISTER(spill_data_dir_metric_entity, spill_disk_limit);
INT_GAUGE_METRIC_REGISTER(spill_data_dir_metric_entity,
spill_disk_avail_capacity);
diff --git a/regression-test/suites/check_before_quit/check_before_quit.groovy
b/regression-test/suites/check_before_quit/check_before_quit.groovy
index 7623ce64545..b5b7803cb16 100644
--- a/regression-test/suites/check_before_quit/check_before_quit.groovy
+++ b/regression-test/suites/check_before_quit/check_before_quit.groovy
@@ -71,4 +71,81 @@ suite("check_before_quit", "nonConcurrent,p0") {
}
assertTrue(clear)
+
+ // check spill temporary files is cleared
+ // Function to parse the metrics
+ def getPrometheusMetric = { String data, String name -> {
+ def metricValues = []
+ data.eachLine { line ->
+ line = line.trim()
+
+ // Skip comment lines
+ if (line.startsWith('#')) return
+
+ // Regular expression to match metric lines
+ def matcher = (line =~ /^(\w+)(\{[^}]+\})?\s+(.+)$/)
+
+ if (matcher) {
+ def metricName = matcher[0][1]
+ def labels = matcher[0][2]
+ def value = matcher[0][3]
+ if (metricName == name) {
+ metricValues << value
+ }
+ }
+ }
+ return metricValues
+ }
+
+ }
+ beginTime = System.currentTimeMillis();
+ timeoutMs = 30 * 1000 // 30s
+ clear = false
+
+ def command = "curl http://${beHost}:${bePort}/metrics";
+ while ((System.currentTimeMillis() - beginTime) < timeoutMs) {
+ clear = true
+ logger.info("executing command: ${command}")
+ def process = command.execute()
+ def outputStream = new StringBuffer()
+ def errorStream = new StringBuffer()
+ process.consumeProcessOutput(outputStream, errorStream)
+ def code = process.waitFor()
+ def metrics = outputStream.toString()
+ logger.info("Request BE metrics: code=" + code + ", err=" +
errorStream.toString())
+
+ def hasSpillData = getPrometheusMetric(metrics,
"doris_be_spill_disk_has_spill_data")
+ logger.info("has spill temporary files :${hasSpillData}")
+ for (int i = 0; i < hasSpillData.size(); i++) {
+ if (0 != Integer.valueOf(hasSpillData.get(i))) {
+ clear = false;
+ break;
+ }
+ }
+
+ hasSpillData = getPrometheusMetric(metrics,
"doris_be_spill_disk_has_spill_gc_data")
+ logger.info("has spill temporary files :${hasSpillData}")
+ for (int i = 0; i < hasSpillData.size(); i++) {
+ if (0 != Integer.valueOf(hasSpillData.get(i))) {
+ clear = false;
+ break;
+ }
+ }
+
+ def spill_data_sizes = getPrometheusMetric(metrics,
"doris_be_spill_disk_data_size")
+ logger.info("spill data sizes :${spill_data_sizes}")
+ for (int i = 0; i < spill_data_sizes.size(); i++) {
+ if (0 != Integer.valueOf(spill_data_sizes.get(i))) {
+ clear = false;
+ break;
+ }
+ }
+
+ if (clear) {
+ break
+ }
+
+ Thread.sleep(2000)
+ }
+ assertTrue(clear)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
