Federico Simoncelli has uploaded a new change for review. Change subject: provider: glance images listing size and limits ......................................................................
provider: glance images listing size and limits This patch adds two configuration parameters to control the glance image listing. In fact OpenStack glance servers by default return a limited number of images (limit_param_default) and it's up to the client to either request a longer list (using the "limit" parameter) or to iterate with multiple requests. GlanceImageListSize defines the number of images that should be returned in each request (20 as the default in glance client). GlanceImageTotalListSize defines the total number of images that should be transferred from the server (with multiple requests if necessary). Change-Id: I7b534caccbdcf121b81571384e3420f1a6efde00 Signed-off-by: Federico Simoncelli <[email protected]> --- M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/IsoDomainListSyncronizer.java M backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/provider/OpenStackImageProviderProxy.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java M backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java M backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogDirector.java M backend/manager/modules/dal/src/main/resources/bundles/AuditLogMessages.properties M packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql 7 files changed, 54 insertions(+), 7 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-engine refs/changes/26/20626/1 diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/IsoDomainListSyncronizer.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/IsoDomainListSyncronizer.java index f516a0e..17778ba 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/IsoDomainListSyncronizer.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/IsoDomainListSyncronizer.java @@ -222,10 +222,23 @@ @Override public Object runInTransaction() { repoFileMetaDataDao.removeRepoDomainFileList(storageDomain.getId(), imageType); - for (RepoImage repoImage : client.getAllImagesAsRepoImages()) { + + Integer totalListSize = Config.<Integer> GetValue(ConfigValues.GlanceImageTotalListSize); + List<RepoImage> repoImages = client.getAllImagesAsRepoImages( + Config.<Integer> GetValue(ConfigValues.GlanceImageListSize), totalListSize); + + if (repoImages.size() >= totalListSize) { + AuditLogableBase logable = new AuditLogableBase(); + logable.addCustomValue("imageDomain", storageDomain.getName()); + logable.addCustomValue("imageListSize", String.valueOf(repoImages.size())); + AuditLogDirector.log(logable, AuditLogType.REFRESH_REPOSITORY_IMAGE_LIST_INCOMPLETE); + } + + for (RepoImage repoImage : repoImages) { repoImage.setRepoDomainId(storageDomain.getId()); repoFileMetaDataDao.addRepoFileMap(repoImage); } + return true; } }); diff --git a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/provider/OpenStackImageProviderProxy.java b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/provider/OpenStackImageProviderProxy.java index 33f747e..50987af 100644 --- a/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/provider/OpenStackImageProviderProxy.java +++ b/backend/manager/modules/bll/src/main/java/org/ovirt/engine/core/bll/provider/OpenStackImageProviderProxy.java @@ -6,6 +6,7 @@ import com.woorea.openstack.glance.Glance; import com.woorea.openstack.glance.model.Image; import com.woorea.openstack.glance.model.ImageDownload; +import com.woorea.openstack.glance.model.Images; import com.woorea.openstack.keystone.utils.KeystoneTokenProvider; import org.ovirt.engine.core.common.businessentities.DiskImage; import org.ovirt.engine.core.common.businessentities.ImageFileType; @@ -213,16 +214,34 @@ return repoImage; } - public List<RepoImage> getAllImagesAsRepoImages() { + public List<RepoImage> getAllImagesAsRepoImages(Integer listSize, Integer totalListSize) { ArrayList<RepoImage> repoImages = new ArrayList<>(); long currentTime = System.currentTimeMillis(); - for (Image glanceImage : getClient().images().list(true).execute()) { - RepoImage repoImage = imageToRepoImage(glanceImage); - repoImage.setLastRefreshed(currentTime); - repoImages.add(repoImage); - } + Images images = null; + + do { + OpenStackRequest<Images> listRequest = getClient().images() + .list(true) + .queryParam("limit", listSize) + .queryParam("sort_key", "name") + .queryParam("sort_dir", "asc"); + + if (images != null) { + listRequest.queryParam("marker", + images.getList().get(images.getList().size() - 1).getId()); + } + + images = listRequest.execute(); + + for (Image glanceImage : images) { + RepoImage repoImage = imageToRepoImage(glanceImage); + repoImage.setLastRefreshed(currentTime); + repoImages.add(repoImage); + } + } while((images.getList().size() >= listSize) && + (totalListSize != null && repoImages.size() < totalListSize)); return repoImages; } diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java index 789a87f..134daa0 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/AuditLogType.java @@ -713,6 +713,7 @@ STORAGE_DOMAIN_TASKS_ERROR(1004), UPDATE_OVF_FOR_STORAGE_POOL_FAILED(1005), UPGRADE_STORAGE_POOL_ENCOUNTERED_PROBLEMS(1006), + REFRESH_REPOSITORY_IMAGE_LIST_INCOMPLETE(1007), RELOAD_CONFIGURATIONS_SUCCESS(1010), RELOAD_CONFIGURATIONS_FAILURE(1011), diff --git a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java index 7f38ffc..b43410a 100644 --- a/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java +++ b/backend/manager/modules/common/src/main/java/org/ovirt/engine/core/common/config/ConfigValues.java @@ -1525,6 +1525,14 @@ @DefaultValueAttribute("true") HotPlugDiskSnapshotSupported(541), + @TypeConverterAttribute(Integer.class) + @DefaultValueAttribute("20") + GlanceImageListSize(542), + + @TypeConverterAttribute(Integer.class) + @DefaultValueAttribute("500") + GlanceImageTotalListSize(543), + Invalid(65535); private int intValue; diff --git a/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogDirector.java b/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogDirector.java index b064bb2..cc2368c 100644 --- a/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogDirector.java +++ b/backend/manager/modules/dal/src/main/java/org/ovirt/engine/core/dal/dbbroker/auditloghandling/AuditLogDirector.java @@ -444,6 +444,7 @@ severities.put(AuditLogType.CONNECT_STORAGE_POOL_FAILED, AuditLogSeverity.WARNING); severities.put(AuditLogType.STORAGE_DOMAIN_ERROR, AuditLogSeverity.ERROR); severities.put(AuditLogType.REFRESH_REPOSITORY_IMAGE_LIST_FAILED, AuditLogSeverity.ERROR); + severities.put(AuditLogType.REFRESH_REPOSITORY_IMAGE_LIST_INCOMPLETE, AuditLogSeverity.WARNING); severities.put(AuditLogType.STORAGE_ALERT_VG_METADATA_CRITICALLY_FULL, AuditLogSeverity.ERROR); severities.put(AuditLogType.STORAGE_ALERT_SMALL_VG_METADATA, AuditLogSeverity.WARNING); severities.put(AuditLogType.STORAGE_ACTIVATE_ASYNC, AuditLogSeverity.NORMAL); diff --git a/backend/manager/modules/dal/src/main/resources/bundles/AuditLogMessages.properties b/backend/manager/modules/dal/src/main/resources/bundles/AuditLogMessages.properties index 764b8be..315c1c7 100644 --- a/backend/manager/modules/dal/src/main/resources/bundles/AuditLogMessages.properties +++ b/backend/manager/modules/dal/src/main/resources/bundles/AuditLogMessages.properties @@ -568,6 +568,7 @@ TASK_STOPPING_ASYNC_TASK=Stopping async task ${CommandName} that started at ${Date} REFRESH_REPOSITORY_IMAGE_LIST_FAILED=Refresh image list failed for domain(s): ${imageDomains}. Please check domain activity. REFRESH_REPOSITORY_IMAGE_LIST_SUCCEEDED=Refresh image list succeeded for domain(s): ${imageDomains} +REFRESH_REPOSITORY_IMAGE_LIST_INCOMPLETE=Refresh image list probably incomplete for domain ${imageDomain}, only ${imageListSize} images discovered. TASK_CLEARING_ASYNC_TASK=Clearing asynchronous task ${CommandName} that started at ${Date} VM_WAS_SET_DOWN_DUE_TO_HOST_REBOOT_OR_MANUAL_FENCE=Vm ${VmName} was shut down due to ${VdsName} host reboot or manual fence UPDATE_TAGS_VM_DEFAULT_DISPLAY_TYPE=Vm ${VmName} tag default display type was updated diff --git a/packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql b/packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql index b773a72..1c890db 100644 --- a/packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql +++ b/packaging/dbscripts/upgrade/pre_upgrade/0000_config.sql @@ -627,6 +627,10 @@ select fn_db_add_config_value('DwhHeartBeatInterval', '30', 'general'); +-- OpenStack Glance +select fn_db_add_config_value('GlanceImageListSize','20','general'); +select fn_db_add_config_value('GlanceImageTotalListSize','500','general'); + ------------------------------------------------------------------------------------ -- Update with override section ------------------------------------------------------------------------------------ -- To view, visit http://gerrit.ovirt.org/20626 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I7b534caccbdcf121b81571384e3420f1a6efde00 Gerrit-PatchSet: 1 Gerrit-Project: ovirt-engine Gerrit-Branch: ovirt-engine-3.3 Gerrit-Owner: Federico Simoncelli <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
