This is an automated email from the ASF dual-hosted git repository.
dahn pushed a commit to branch 4.19
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/4.19 by this push:
new ae3fa5d0de3 Add configuration to limit the number of rows deleted from
vm_stats (#8740)
ae3fa5d0de3 is described below
commit ae3fa5d0de3407a63e173cb367af3b517691039d
Author: João Jandre <[email protected]>
AuthorDate: Thu Jun 20 09:26:36 2024 -0300
Add configuration to limit the number of rows deleted from vm_stats (#8740)
Co-authored-by: João Jandre <[email protected]>
---
.../src/main/java/com/cloud/vm/dao/VmStatsDao.java | 6 ++++--
.../main/java/com/cloud/vm/dao/VmStatsDaoImpl.java | 21 ++++++++++++++++++---
.../main/java/com/cloud/utils/db/GenericDao.java | 8 ++++++++
.../java/com/cloud/utils/db/GenericDaoBase.java | 12 +++++++++++-
.../main/java/com/cloud/server/StatsCollector.java | 12 +++++++-----
.../java/com/cloud/server/StatsCollectorTest.java | 8 ++++----
.../test/java/com/cloud/user/MockUsageEventDao.java | 5 +++++
7 files changed, 57 insertions(+), 15 deletions(-)
diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDao.java
b/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDao.java
index 879faaf5c90..0d7aa703a8c 100644
--- a/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDao.java
+++ b/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDao.java
@@ -75,8 +75,10 @@ public interface VmStatsDao extends GenericDao<VmStatsVO,
Long> {
/**
* Removes (expunges) all VM stats with {@code timestamp} less than
* a given Date.
- * @param limit the maximum date to keep stored. Records that exceed this
limit will be removed.
+ * @param limitDate the maximum date to keep stored. Records that exceed
this limit will be removed.
+ * @param limitPerQuery the maximum amount of rows to be removed in a
single query. We loop if there are still rows to be removed after a given query.
+ * If 0 or negative, no limit is used.
*/
- void removeAllByTimestampLessThan(Date limit);
+ void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery);
}
diff --git a/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java
b/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java
index 1bef8f0626c..a98302e2136 100644
--- a/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java
+++ b/engine/schema/src/main/java/com/cloud/vm/dao/VmStatsDaoImpl.java
@@ -21,6 +21,7 @@ import java.util.List;
import javax.annotation.PostConstruct;
+import org.apache.log4j.Logger;
import org.springframework.stereotype.Component;
import com.cloud.utils.db.Filter;
@@ -33,6 +34,8 @@ import com.cloud.vm.VmStatsVO;
@Component
public class VmStatsDaoImpl extends GenericDaoBase<VmStatsVO, Long> implements
VmStatsDao {
+ protected Logger logger = Logger.getLogger(getClass());
+
protected SearchBuilder<VmStatsVO> vmIdSearch;
protected SearchBuilder<VmStatsVO> vmIdTimestampGreaterThanEqualSearch;
protected SearchBuilder<VmStatsVO> vmIdTimestampLessThanEqualSearch;
@@ -113,10 +116,22 @@ public class VmStatsDaoImpl extends
GenericDaoBase<VmStatsVO, Long> implements V
}
@Override
- public void removeAllByTimestampLessThan(Date limit) {
+ public void removeAllByTimestampLessThan(Date limitDate, long
limitPerQuery) {
SearchCriteria<VmStatsVO> sc = timestampSearch.create();
- sc.setParameters("timestamp", limit);
- expunge(sc);
+ sc.setParameters("timestamp", limitDate);
+
+ logger.debug(String.format("Starting to remove all vm_stats rows older
than [%s].", limitDate));
+
+ long totalRemoved = 0;
+ long removed;
+
+ do {
+ removed = expunge(sc, limitPerQuery);
+ totalRemoved += removed;
+ logger.trace(String.format("Removed [%s] vm_stats rows on the last
update and a sum of [%s] vm_stats rows older than [%s] until now.", removed,
totalRemoved, limitDate));
+ } while (limitPerQuery > 0 && removed >= limitPerQuery);
+
+ logger.info(String.format("Removed a total of [%s] vm_stats rows older
than [%s].", totalRemoved, limitDate));
}
}
diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java
b/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java
index 2fc02301cb7..b9199468bd1 100644
--- a/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java
+++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDao.java
@@ -229,6 +229,14 @@ public interface GenericDao<T, ID extends Serializable> {
*/
int expunge(final SearchCriteria<T> sc);
+ /**
+ * Delete the entity beans specified by the search criteria with a given
limit
+ * @param sc Search criteria
+ * @param limit Maximum number of rows that will be affected
+ * @return Number of rows deleted
+ */
+ int expunge(SearchCriteria<T> sc, long limit);
+
/**
* expunge the removed rows.
*/
diff --git a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java
b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java
index 0eb45439769..3b950e1983d 100644
--- a/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java
+++ b/framework/db/src/main/java/com/cloud/utils/db/GenericDaoBase.java
@@ -1226,9 +1226,14 @@ public abstract class GenericDaoBase<T, ID extends
Serializable> extends Compone
}
}
- // FIXME: Does not work for joins.
@Override
public int expunge(final SearchCriteria<T> sc) {
+ return expunge(sc, -1);
+ }
+
+ // FIXME: Does not work for joins.
+ @Override
+ public int expunge(final SearchCriteria<T> sc, long limit) {
if (sc == null) {
throw new CloudRuntimeException("Call to throw new expunge with
null search Criteria");
}
@@ -1241,6 +1246,11 @@ public abstract class GenericDaoBase<T, ID extends
Serializable> extends Compone
str.append(sc.getWhereClause());
}
+ if (limit > 0) {
+ str.append(" LIMIT ");
+ str.append(limit);
+ }
+
final String sql = str.toString();
final TransactionLegacy txn = TransactionLegacy.currentTxn();
diff --git a/server/src/main/java/com/cloud/server/StatsCollector.java
b/server/src/main/java/com/cloud/server/StatsCollector.java
index 5aa6dd0a214..070508f2502 100644
--- a/server/src/main/java/com/cloud/server/StatsCollector.java
+++ b/server/src/main/java/com/cloud/server/StatsCollector.java
@@ -114,9 +114,6 @@ import com.cloud.org.Cluster;
import com.cloud.resource.ResourceManager;
import com.cloud.resource.ResourceState;
import com.cloud.serializer.GsonHelper;
-import com.cloud.server.StatsCollector.AbstractStatsCollector;
-import com.cloud.server.StatsCollector.AutoScaleMonitor;
-import com.cloud.server.StatsCollector.StorageCollector;
import com.cloud.storage.ImageStoreDetailsUtil;
import com.cloud.storage.ScopeType;
import com.cloud.storage.Storage;
@@ -287,6 +284,11 @@ public class StatsCollector extends ManagerBase implements
ComponentMethodInterc
protected static ConfigKey<Boolean> vmStatsCollectUserVMOnly = new
ConfigKey<>("Advanced", Boolean.class, "vm.stats.user.vm.only", "false",
"When set to 'false' stats for system VMs will be collected
otherwise stats collection will be done only for user VMs", true);
+ protected static ConfigKey<Long> vmStatsRemoveBatchSize = new
ConfigKey<>("Advanced", Long.class, "vm.stats.remove.batch.size", "0",
"Indicates the" +
+ " limit applied to delete vm_stats entries while running the
clean-up task. With this, ACS will run the delete query, applying the limit, as
many times as necessary" +
+ " to delete all entries older than the value defined in
vm.stats.max.retention.time. This is advised when retaining several days of
records, which can lead to slowness" +
+ " on the delete query. Zero (0) means that no limit will be
applied, therefore, the query will run once and without limit, keeping the
default behavior.", true);
+
protected static ConfigKey<Boolean> vmDiskStatsRetentionEnabled = new
ConfigKey<>("Advanced", Boolean.class, "vm.disk.stats.retention.enabled",
"false",
"When set to 'true' stats for VM disks will be stored in the
database otherwise disk stats will not be stored", true);
@@ -1965,7 +1967,7 @@ public class StatsCollector extends ManagerBase
implements ComponentMethodInterc
LOGGER.trace("Removing older VM stats records.");
Date now = new Date();
Date limit = DateUtils.addMinutes(now, -maxRetentionTime);
- vmStatsDao.removeAllByTimestampLessThan(limit);
+ vmStatsDao.removeAllByTimestampLessThan(limit,
vmStatsRemoveBatchSize.value());
}
/**
@@ -2139,7 +2141,7 @@ public class StatsCollector extends ManagerBase
implements ComponentMethodInterc
@Override
public ConfigKey<?>[] getConfigKeys() {
- return new ConfigKey<?>[] {vmDiskStatsInterval,
vmDiskStatsIntervalMin, vmNetworkStatsInterval, vmNetworkStatsIntervalMin,
StatsTimeout, statsOutputUri,
+ return new ConfigKey<?>[] {vmDiskStatsInterval,
vmDiskStatsIntervalMin, vmNetworkStatsInterval, vmNetworkStatsIntervalMin,
StatsTimeout, statsOutputUri, vmStatsRemoveBatchSize,
vmStatsIncrementMetrics, vmStatsMaxRetentionTime,
vmStatsCollectUserVMOnly, vmDiskStatsRetentionEnabled,
vmDiskStatsMaxRetentionTime,
MANAGEMENT_SERVER_STATUS_COLLECTION_INTERVAL,
DATABASE_SERVER_STATUS_COLLECTION_INTERVAL,
diff --git a/server/src/test/java/com/cloud/server/StatsCollectorTest.java
b/server/src/test/java/com/cloud/server/StatsCollectorTest.java
index 1f6a35cfbae..2b2451c66c7 100644
--- a/server/src/test/java/com/cloud/server/StatsCollectorTest.java
+++ b/server/src/test/java/com/cloud/server/StatsCollectorTest.java
@@ -28,7 +28,7 @@ import com.cloud.user.VmDiskStatisticsVO;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.vm.VmStats;
import com.cloud.vm.VmStatsVO;
-import com.cloud.vm.dao.VmStatsDao;
+import com.cloud.vm.dao.VmStatsDaoImpl;
import com.google.gson.Gson;
import com.tngtech.java.junit.dataprovider.DataProvider;
import com.tngtech.java.junit.dataprovider.DataProviderRunner;
@@ -81,7 +81,7 @@ public class StatsCollectorTest {
private static final String DEFAULT_DATABASE_NAME = "cloudstack";
@Mock
- VmStatsDao vmStatsDaoMock = Mockito.mock(VmStatsDao.class);
+ VmStatsDaoImpl vmStatsDaoMock;
@Mock
VmStatsEntry statsForCurrentIterationMock;
@@ -304,7 +304,7 @@ public class StatsCollectorTest {
statsCollector.cleanUpVirtualMachineStats();
- Mockito.verify(vmStatsDaoMock,
Mockito.never()).removeAllByTimestampLessThan(Mockito.any());
+ Mockito.verify(vmStatsDaoMock,
Mockito.never()).removeAllByTimestampLessThan(Mockito.any(), Mockito.anyLong());
}
@Test
@@ -313,7 +313,7 @@ public class StatsCollectorTest {
statsCollector.cleanUpVirtualMachineStats();
-
Mockito.verify(vmStatsDaoMock).removeAllByTimestampLessThan(Mockito.any());
+
Mockito.verify(vmStatsDaoMock).removeAllByTimestampLessThan(Mockito.any(),
Mockito.anyLong());
}
@Test
diff --git a/server/src/test/java/com/cloud/user/MockUsageEventDao.java
b/server/src/test/java/com/cloud/user/MockUsageEventDao.java
index 52e1b1a02b4..4769126fdcb 100644
--- a/server/src/test/java/com/cloud/user/MockUsageEventDao.java
+++ b/server/src/test/java/com/cloud/user/MockUsageEventDao.java
@@ -210,6 +210,11 @@ public class MockUsageEventDao implements UsageEventDao{
return 0;
}
+ @Override
+ public int expunge(SearchCriteria<UsageEventVO> sc, long limit) {
+ return 0;
+ }
+
@Override
public void expunge() {
