[
https://issues.apache.org/jira/browse/CLOUDSTACK-10241?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16409568#comment-16409568
]
ASF GitHub Bot commented on CLOUDSTACK-10241:
---------------------------------------------
GabrielBrascher commented on a change in pull request #2414: [CLOUDSTACK-10241]
Duplicated file SRs being created in XenServer pools
URL: https://github.com/apache/cloudstack/pull/2414#discussion_r175882326
##########
File path:
plugins/hypervisors/xenserver/src/main/java/com/cloud/hypervisor/xenserver/resource/Xenserver625StorageProcessor.java
##########
@@ -65,90 +69,181 @@ public Xenserver625StorageProcessor(final
CitrixResourceBase resource) {
super(resource);
}
- protected boolean mountNfs(final Connection conn, final String remoteDir,
String localDir) {
+ private void mountNfs(Connection conn, String remoteDir, String localDir) {
if (localDir == null) {
localDir = "/var/cloud_mount/" +
UUID.nameUUIDFromBytes(remoteDir.getBytes());
}
-
- final String results = hypervisorResource.callHostPluginAsync(conn,
"cloud-plugin-storage", "mountNfsSecondaryStorage", 100 * 1000, "localDir",
localDir, "remoteDir",
- remoteDir);
-
- if (results == null || results.isEmpty()) {
+ String result = hypervisorResource.callHostPluginAsync(conn,
"cloud-plugin-storage", "mountNfsSecondaryStorage", 100 * 1000, "localDir",
localDir, "remoteDir", remoteDir);
+ if (StringUtils.isBlank(result)) {
final String errMsg = "Could not mount secondary storage " +
remoteDir + " on host " + localDir;
-
s_logger.warn(errMsg);
-
throw new CloudRuntimeException(errMsg);
}
-
- return true;
}
- protected boolean makeDirectory(final Connection conn, final String path) {
- final String result = hypervisorResource.callHostPlugin(conn,
"cloud-plugin-storage", "makeDirectory", "path", path);
+ protected boolean makeDirectory(Connection conn, String path) {
+ String result = hypervisorResource.callHostPlugin(conn,
"cloud-plugin-storage", "makeDirectory", "path", path);
+ return StringUtils.isNotBlank(result);
+ }
- if (result == null || result.isEmpty()) {
- return false;
+ /**
+ * Creates the file SR for the given path. If there already exist a file
SR for the path, we return the existing one.
+ * This method uses a synchronized block to guarantee that only a single
file SR is created per path.
+ * If it is not possible to retrieve one file SR or to create one, a
runtime exception will be thrown.
+ */
+ protected SR createFileSR(Connection conn, String path) {
+ String srPath = StringUtils.trim(path);
+ synchronized (srPath) {
+ SR sr = retrieveAlreadyConfiguredSrWithoutException(conn, srPath);
+ if (sr == null) {
+ sr = createNewFileSr(conn, srPath);
+ }
+ if (sr == null) {
+ String hostUuid = this.hypervisorResource._host.getUuid();
+ throw new CloudRuntimeException(String.format("Could not
retrieve an already used file SR for path [%s] or create a new file SR on host
[%s]", srPath, hostUuid));
+ }
+ return sr;
}
-
- return true;
}
- protected SR createFileSR(final Connection conn, final String path) {
+ /**
+ * Creates a new file SR for the given path. If any of XenServer's checked
exception occurs, we use method {@link #removeSrAndPbdIfPossible(Connection,
SR, PBD)} to clean the created PBD and SR entries.
+ * To avoid race conditions between management servers, we are using a
deterministic srUuid for the file SR to be created (we are leaving XenServer
with the burden of managing race conditions). The UUID is based on the SR file
path, and is generated using {@link UUID#nameUUIDFromBytes(byte[])}.
+ * If there is an SR with the generated UUID, this means that some other
management server has just created it. An exception will occur and this
exception will be an {@link InternalError}. The exception will contain {@link
InternalError#message} a message saying
'Db_exn.Uniqueness_constraint_violation'.
+ * For cases where the previous described error happens, we catch the
exception and use the method {@link
#retrieveAlreadyConfiguredSrWithoutException(Connection, String)}.
+ */
+ protected SR createNewFileSr(Connection conn, String srPath) {
+ String hostUuid = hypervisorResource.getHost().getUuid();
+ s_logger.debug(String.format("Creating file SR for path [%s] on host
[%s]", srPath, this.hypervisorResource._host.getUuid()));
SR sr = null;
PBD pbd = null;
-
try {
- final String srname = path.trim();
- synchronized (srname.intern()) {
- final Set<SR> srs = SR.getByNameLabel(conn, srname);
- if (srs != null && !srs.isEmpty()) {
- return srs.iterator().next();
- }
- final Map<String, String> smConfig = new HashMap<String,
String>();
- final Host host = Host.getByUuid(conn,
hypervisorResource.getHost().getUuid());
- final String uuid = UUID.randomUUID().toString();
- sr = SR.introduce(conn, uuid, srname, srname, "file", "file",
false, smConfig);
- final PBD.Record record = new PBD.Record();
- record.host = host;
- record.SR = sr;
- smConfig.put("location", path);
- record.deviceConfig = smConfig;
- pbd = PBD.create(conn, record);
- pbd.plug(conn);
- sr.scan(conn);
- }
+ Host host = Host.getByUuid(conn, hostUuid);
+ String srUuid =
UUID.nameUUIDFromBytes(srPath.getBytes()).toString();
+
+ Map<String, String> smConfig = new HashMap<String, String>();
+ sr = SR.introduce(conn, srUuid, srPath, srPath, "file", "file",
false, smConfig);
+
+ PBD.Record record = new PBD.Record();
+ record.host = host;
+ record.SR = sr;
+ smConfig.put("location", srPath);
+ record.deviceConfig = smConfig;
+ pbd = PBD.create(conn, record);
+ pbd.plug(conn);
+ sr.scan(conn);
return sr;
- } catch (final Exception ex) {
- try {
- if (pbd != null) {
- pbd.destroy(conn);
+ } catch (XenAPIException | XmlRpcException e) {
+ if (e instanceof Types.InternalError) {
+ String expectedDuplicatedFileSrErrorMessage =
"Db_exn.Uniqueness_constraint_violation";
+
+ Types.InternalError internalErrorException =
(Types.InternalError)e;
+ if (StringUtils.contains(internalErrorException.message,
expectedDuplicatedFileSrErrorMessage)) {
+ s_logger.debug(String.format(
+ "It seems that we have hit a race condition case
here while creating file SR for [%s]. Instead of creating one, we will reuse
the one that already exist in the XenServer pool.",
+ srPath));
+ return retrieveAlreadyConfiguredSrWithoutException(conn,
srPath);
}
- } catch (final Exception e1) {
- s_logger.debug("Failed to destroy PBD", ex);
}
+ removeSrAndPbdIfPossible(conn, sr, pbd);
+ s_logger.debug(String.format("Could not create file SR [%s] on
host [%s].", srPath, hostUuid), e);
+ return null;
+ }
+ }
- try {
- if (sr != null) {
- sr.forget(conn);
- }
- } catch (final Exception e2) {
- s_logger.error("Failed to forget SR", ex);
- }
+ /**
+ * Calls {@link #unplugPbd(Connection, PBD)} and {@link
#forgetSr(Connection, SR)}, if respective objects are not null.
+ */
+ protected void removeSrAndPbdIfPossible(Connection conn, SR sr, PBD pbd) {
+ if (pbd != null) {
+ unplugPbd(conn, pbd);
+ }
+ if (sr != null) {
+ forgetSr(conn, sr);
+ }
+ }
+
+ /**
+ * This is a simple facade for {@link
#retrieveAlreadyConfiguredSr(Connection, String)} method.
+ * If we catch any of the checked exception of {@link
#retrieveAlreadyConfiguredSr(Connection, String)}, we re-throw as a {@link
CloudRuntimeException}.
+ */
+ protected SR retrieveAlreadyConfiguredSrWithoutException(Connection conn,
String srPath) {
+ try {
+ return retrieveAlreadyConfiguredSr(conn, srPath);
+ } catch (XenAPIException | XmlRpcException e) {
+ throw new CloudRuntimeException("Unexpected exception while trying
to retrieve an already configured file SR for path: " + srPath);
+ }
+ }
- final String msg = "createFileSR failed! due to the following: " +
ex.toString();
+ /**
+ * This method will check if there is an already configured file SR for
the given path. If by any chance we find more an one SR with the same name
(mount point path) we throw a runtime exception because this situation should
never happen.
Review comment:
If by any chance we find more **an** one SR with the same name -> more
**than** one
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
> Duplicated file SRs being created in XenServer pools
> ----------------------------------------------------
>
> Key: CLOUDSTACK-10241
> URL: https://issues.apache.org/jira/browse/CLOUDSTACK-10241
> Project: CloudStack
> Issue Type: Bug
> Security Level: Public(Anyone can view this level - this is the
> default.)
> Reporter: Rafael Weingärtner
> Assignee: Rafael Weingärtner
> Priority: Major
>
> Due to a race condition between multiple management servers, in some rare
> cases, CloudStack is creating multiple file SRs to the same secondary folder.
> This causes a problem when introducing the SR to the XenServer pools, as
> “there will be VDIs with duplicated UUIDs“. The VDIs are the same, but they
> are seen in different SRs, and therefore cause an error.
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
