This is an automated email from the ASF dual-hosted git repository.
rohit pushed a commit to branch 4.11
in repository https://gitbox.apache.org/repos/asf/cloudstack.git
The following commit(s) were added to refs/heads/4.11 by this push:
new b6d420b CLOUDSTACK-9677: Adding storage policy support for swift as
secondary storage (#2412)
b6d420b is described below
commit b6d420bec3b4cb02651cffd43502ed880f100bef
Author: Khosrow Moossavi <[email protected]>
AuthorDate: Wed Apr 25 18:42:15 2018 -0400
CLOUDSTACK-9677: Adding storage policy support for swift as secondary
storage (#2412)
Original-Author: @pdube on PR Fixes #1830.
---
api/src/com/cloud/agent/api/to/SwiftTO.java | 9 +-
.../org/apache/cloudstack/api/ApiConstants.java | 1 +
.../resource/XenServerStorageProcessor.java | 115 +++++++++----
.../resource/XenServerStorageProcessorTest.java | 113 +++++++++++++
.../driver/SwiftImageStoreDriverImpl.java | 2 +-
scripts/storage/secondary/swift | 21 ++-
scripts/vm/hypervisor/xenserver/swift | 21 ++-
ui/l10n/en.js | 1 +
ui/scripts/system.js | 29 +++-
utils/src/main/java/com/cloud/utils/SwiftUtil.java | 187 +++++++++++----------
.../test/java/com/cloud/utils/SwiftUtilTest.java | 150 ++++++++++++++++-
11 files changed, 499 insertions(+), 150 deletions(-)
diff --git a/api/src/com/cloud/agent/api/to/SwiftTO.java
b/api/src/com/cloud/agent/api/to/SwiftTO.java
index c7a9866..b89dfea 100644
--- a/api/src/com/cloud/agent/api/to/SwiftTO.java
+++ b/api/src/com/cloud/agent/api/to/SwiftTO.java
@@ -26,17 +26,19 @@ public class SwiftTO implements DataStoreTO,
SwiftUtil.SwiftClientCfg {
String userName;
String key;
+ String storagePolicy;
private static final String pathSeparator = "/";
public SwiftTO() {
}
- public SwiftTO(Long id, String url, String account, String userName,
String key) {
+ public SwiftTO(Long id, String url, String account, String userName,
String key, String storagePolicy) {
this.id = id;
this.url = url;
this.account = account;
this.userName = userName;
this.key = key;
+ this.storagePolicy = storagePolicy;
}
public Long getId() {
@@ -64,6 +66,11 @@ public class SwiftTO implements DataStoreTO,
SwiftUtil.SwiftClientCfg {
}
@Override
+ public String getStoragePolicy() {
+ return this.storagePolicy;
+ }
+
+ @Override
public DataStoreRole getRole() {
return DataStoreRole.Image;
}
diff --git a/api/src/org/apache/cloudstack/api/ApiConstants.java
b/api/src/org/apache/cloudstack/api/ApiConstants.java
index d9090b8..dfe9b30 100644
--- a/api/src/org/apache/cloudstack/api/ApiConstants.java
+++ b/api/src/org/apache/cloudstack/api/ApiConstants.java
@@ -287,6 +287,7 @@ public class ApiConstants {
public static final String STATE = "state";
public static final String STATUS = "status";
public static final String STORAGE_TYPE = "storagetype";
+ public static final String STORAGE_POLICY = "storagepolicy";
public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled";
public static final String STORAGE_CAPABILITIES = "storagecapabilities";
public static final String SYSTEM_VM_TYPE = "systemvmtype";
diff --git
a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
index b575fdb..d2ce43e 100644
---
a/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
+++
b/plugins/hypervisors/xenserver/src/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessor.java
@@ -18,23 +18,24 @@
*/
package com.cloud.hypervisor.xenserver.resource;
-import com.cloud.agent.api.Answer;
-import com.cloud.agent.api.to.DataObjectType;
-import com.cloud.agent.api.to.DataStoreTO;
-import com.cloud.agent.api.to.DataTO;
-import com.cloud.agent.api.to.DiskTO;
-import com.cloud.agent.api.to.NfsTO;
-import com.cloud.agent.api.to.S3TO;
-import com.cloud.agent.api.to.SwiftTO;
-import com.cloud.exception.InternalErrorException;
-import com.cloud.hypervisor.Hypervisor.HypervisorType;
-import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType;
-import com.cloud.storage.DataStoreRole;
-import com.cloud.storage.Storage;
-import com.cloud.storage.Storage.ImageFormat;
-import com.cloud.storage.resource.StorageProcessor;
-import com.cloud.utils.exception.CloudRuntimeException;
-import com.cloud.utils.storage.S3.ClientOptions;
+import static com.cloud.utils.ReflectUtil.flattenProperties;
+import static com.google.common.collect.Lists.newArrayList;
+
+import java.io.File;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.commons.lang3.BooleanUtils;
+import org.apache.log4j.Logger;
+import org.apache.xmlrpc.XmlRpcException;
+
+import com.google.common.annotations.VisibleForTesting;
import com.xensource.xenapi.Connection;
import com.xensource.xenapi.SR;
import com.xensource.xenapi.Types;
@@ -44,6 +45,7 @@ import com.xensource.xenapi.Types.XenAPIException;
import com.xensource.xenapi.VBD;
import com.xensource.xenapi.VDI;
import com.xensource.xenapi.VM;
+
import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand;
import org.apache.cloudstack.storage.command.AttachAnswer;
import org.apache.cloudstack.storage.command.AttachCommand;
@@ -65,20 +67,24 @@ import org.apache.cloudstack.storage.to.PrimaryDataStoreTO;
import org.apache.cloudstack.storage.to.SnapshotObjectTO;
import org.apache.cloudstack.storage.to.TemplateObjectTO;
import org.apache.cloudstack.storage.to.VolumeObjectTO;
-import org.apache.log4j.Logger;
-import org.apache.xmlrpc.XmlRpcException;
-
-import java.io.File;
-import java.net.URI;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import static com.cloud.utils.ReflectUtil.flattenProperties;
-import static com.google.common.collect.Lists.newArrayList;
+import com.cloud.agent.api.Answer;
+import com.cloud.agent.api.to.DataObjectType;
+import com.cloud.agent.api.to.DataStoreTO;
+import com.cloud.agent.api.to.DataTO;
+import com.cloud.agent.api.to.DiskTO;
+import com.cloud.agent.api.to.NfsTO;
+import com.cloud.agent.api.to.S3TO;
+import com.cloud.agent.api.to.SwiftTO;
+import com.cloud.exception.InternalErrorException;
+import com.cloud.hypervisor.Hypervisor.HypervisorType;
+import com.cloud.hypervisor.xenserver.resource.CitrixResourceBase.SRType;
+import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.Storage;
+import com.cloud.storage.Storage.ImageFormat;
+import com.cloud.storage.resource.StorageProcessor;
+import com.cloud.utils.exception.CloudRuntimeException;
+import com.cloud.utils.storage.S3.ClientOptions;
public class XenServerStorageProcessor implements StorageProcessor {
private static final Logger s_logger =
Logger.getLogger(XenServerStorageProcessor.class);
@@ -914,20 +920,55 @@ public class XenServerStorageProcessor implements
StorageProcessor {
private boolean swiftUpload(final Connection conn, final SwiftTO swift,
final String container, final String ldir, final String lfilename, final
Boolean isISCSI,
final int wait) {
- String result = null;
+
+ List<String> params = getSwiftParams(swift, container, ldir,
lfilename, isISCSI);
+
try {
- result =
- hypervisorResource.callHostPluginAsync(conn,
"swiftxenserver", "swift", wait, "op", "upload", "url", swift.getUrl(),
"account", swift.getAccount(), "username",
- swift.getUserName(), "key", swift.getKey(),
"container", container, "ldir", ldir, "lfilename", lfilename, "isISCSI",
isISCSI.toString());
- if (result != null && result.equals("true")) {
- return true;
- }
+ String result = hypervisorResource.callHostPluginAsync(conn,
"swiftxenserver", "swift", wait, params.toArray(new String[params.size()]));
+ return BooleanUtils.toBoolean(result);
} catch (final Exception e) {
s_logger.warn("swift upload failed due to " + e.toString(), e);
}
return false;
}
+ @VisibleForTesting
+ List<String> getSwiftParams(SwiftTO swift, String container, String ldir,
String lfilename, Boolean isISCSI) {
+ // ORDER IS IMPORTANT
+ List<String> params = new ArrayList<>();
+
+ //operation
+ params.add("op");
+ params.add("upload");
+
+ //auth
+ params.add("url");
+ params.add(swift.getUrl());
+ params.add("account");
+ params.add(swift.getAccount());
+ params.add("username");
+ params.add(swift.getUserName());
+ params.add("key");
+ params.add(swift.getKey());
+
+ // object info
+ params.add("container");
+ params.add(container);
+ params.add("ldir");
+ params.add(ldir);
+ params.add("lfilename");
+ params.add(lfilename);
+ params.add("isISCSI");
+ params.add(isISCSI.toString());
+
+ if (swift.getStoragePolicy() != null) {
+ params.add("storagepolicy");
+ params.add(swift.getStoragePolicy());
+ }
+
+ return params;
+ }
+
protected String deleteSnapshotBackup(final Connection conn, final String
localMountPoint, final String path, final String secondaryStorageMountPath,
final String backupUUID) {
// If anybody modifies the formatting below again, I'll skin them
diff --git
a/plugins/hypervisors/xenserver/test/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessorTest.java
b/plugins/hypervisors/xenserver/test/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessorTest.java
new file mode 100644
index 0000000..75ac8b8
--- /dev/null
+++
b/plugins/hypervisors/xenserver/test/com/cloud/hypervisor/xenserver/resource/XenServerStorageProcessorTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package com.cloud.hypervisor.xenserver.resource;
+
+import static org.mockito.Mockito.when;
+
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import com.cloud.agent.api.to.SwiftTO;
+
+public class XenServerStorageProcessorTest {
+ @Test
+ public void testOrderOfSwiftUplodScriptParamsWithoutStoragePolicy() {
+ CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class);
+ XenServerStorageProcessor mock = new
XenServerStorageProcessor(resource);
+
+ SwiftTO swift = Mockito.mock(SwiftTO.class);
+ when(swift.getStoragePolicy()).thenReturn(null);
+
+ String container = "sample-container-name";
+ String ldir = "sample-ldir";
+ String lfilename = "sample-lfilename";
+ Boolean isISCSI = true;
+
+ List<String> params = mock.getSwiftParams(swift, container, ldir,
lfilename, isISCSI);
+
+ // make sure the params not null and has correct number of items in it
+ Assert.assertNotNull("params is null", params);
+ Assert.assertTrue("Expected param list size is 18 but it was" +
params.size(), params.size() == 18);
+
+ // check the order of params
+ Assert.assertEquals("unexpected param.", "op", params.get(0));
+ Assert.assertEquals("unexpected param.", "upload", params.get(1));
+ Assert.assertEquals("unexpected param.", "url", params.get(2));
+ Assert.assertEquals("unexpected param.", swift.getUrl(),
params.get(3));
+ Assert.assertEquals("unexpected param.", "account", params.get(4));
+ Assert.assertEquals("unexpected param.", swift.getAccount(),
params.get(5));
+ Assert.assertEquals("unexpected param.", "username", params.get(6));
+ Assert.assertEquals("unexpected param.", swift.getUserName(),
params.get(7));
+ Assert.assertEquals("unexpected param.", "key", params.get(8));
+ Assert.assertEquals("unexpected param.", swift.getKey(),
params.get(9));
+ Assert.assertEquals("unexpected param.", "container", params.get(10));
+ Assert.assertEquals("unexpected param.", container, params.get(11));
+ Assert.assertEquals("unexpected param.", "ldir", params.get(12));
+ Assert.assertEquals("unexpected param.", ldir, params.get(13));
+ Assert.assertEquals("unexpected param.", "lfilename", params.get(14));
+ Assert.assertEquals("unexpected param.", lfilename, params.get(15));
+ Assert.assertEquals("unexpected param.", "isISCSI", params.get(16));
+ Assert.assertEquals("unexpected param.", isISCSI.toString(),
params.get(17));
+ }
+
+ @Test
+ public void testOrderOfSwiftUplodScriptParamsWithStoragePolicy() {
+ CitrixResourceBase resource = Mockito.mock(CitrixResourceBase.class);
+ XenServerStorageProcessor mock = new
XenServerStorageProcessor(resource);
+
+ SwiftTO swift = Mockito.mock(SwiftTO.class);
+ when(swift.getStoragePolicy()).thenReturn("sample-storagepolicy");
+
+ String container = "sample-container-name";
+ String ldir = "sample-ldir";
+ String lfilename = "sample-lfilename";
+ Boolean isISCSI = true;
+
+ List<String> params = mock.getSwiftParams(swift, container, ldir,
lfilename, isISCSI);
+
+ // make sure the params not null and has correct number of items in it
+ Assert.assertNotNull("params is null", params);
+ Assert.assertTrue("Expected param list size is 20 but it was" +
params.size(), params.size() == 20);
+
+ // check the order of params
+ Assert.assertEquals("unexpected param.", "op", params.get(0));
+ Assert.assertEquals("unexpected param.", "upload", params.get(1));
+ Assert.assertEquals("unexpected param.", "url", params.get(2));
+ Assert.assertEquals("unexpected param.", swift.getUrl(),
params.get(3));
+ Assert.assertEquals("unexpected param.", "account", params.get(4));
+ Assert.assertEquals("unexpected param.", swift.getAccount(),
params.get(5));
+ Assert.assertEquals("unexpected param.", "username", params.get(6));
+ Assert.assertEquals("unexpected param.", swift.getUserName(),
params.get(7));
+ Assert.assertEquals("unexpected param.", "key", params.get(8));
+ Assert.assertEquals("unexpected param.", swift.getKey(),
params.get(9));
+ Assert.assertEquals("unexpected param.", "container", params.get(10));
+ Assert.assertEquals("unexpected param.", container, params.get(11));
+ Assert.assertEquals("unexpected param.", "ldir", params.get(12));
+ Assert.assertEquals("unexpected param.", ldir, params.get(13));
+ Assert.assertEquals("unexpected param.", "lfilename", params.get(14));
+ Assert.assertEquals("unexpected param.", lfilename, params.get(15));
+ Assert.assertEquals("unexpected param.", "isISCSI", params.get(16));
+ Assert.assertEquals("unexpected param.", isISCSI.toString(),
params.get(17));
+ Assert.assertEquals("unexpected param.", "storagepolicy",
params.get(18));
+ Assert.assertEquals("unexpected param.", "sample-storagepolicy",
params.get(19));
+ }
+}
diff --git
a/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
b/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
index 2816c60..7e14862 100644
---
a/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
+++
b/plugins/storage/image/swift/src/org/apache/cloudstack/storage/datastore/driver/SwiftImageStoreDriverImpl.java
@@ -68,7 +68,7 @@ public class SwiftImageStoreDriverImpl extends
BaseImageStoreDriverImpl {
public DataStoreTO getStoreTO(DataStore store) {
ImageStoreImpl imgStore = (ImageStoreImpl)store;
Map<String, String> details =
_imageStoreDetailsDao.getDetails(imgStore.getId());
- return new SwiftTO(imgStore.getId(), imgStore.getUri(),
details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME),
details.get(ApiConstants.KEY));
+ return new SwiftTO(imgStore.getId(), imgStore.getUri(),
details.get(ApiConstants.ACCOUNT), details.get(ApiConstants.USERNAME),
details.get(ApiConstants.KEY), details.get(ApiConstants.STORAGE_POLICY));
}
@Override
diff --git a/scripts/storage/secondary/swift b/scripts/storage/secondary/swift
index 4138db8..c09c530 100755
--- a/scripts/storage/secondary/swift
+++ b/scripts/storage/secondary/swift
@@ -1473,8 +1473,9 @@ post [options] [container] [object]
Updates meta information for the account, container, or object depending on
the args given. If the container is not found, it will be created
automatically; but this is not true for accounts and objects. Containers
- also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
- or --meta option is allowed on all and used to define the user meta data
+ also allow the -r (or --read-acl) and -w (or --write-acl) options.
+ The --storage-policy will set a storage policy to the container if the
container does not exist.
+ The -m or --meta option is allowed on all and used to define the user meta
data
items to set in the form Name:Value. This option can be repeated. Example:
post -m Color:Blue -m Size:Large'''.strip('\n')
@@ -1493,6 +1494,8 @@ def st_post(options, args, print_queue, error_queue):
parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
help='Sets a meta data item with the syntax name:value. This option '
'may be repeated. Example: -m Color:Blue -m Size:Large')
+ parser.add_option('', '--storage-policy', action='store',
dest='storage_policy',
+ help='Sets a storage policy to the container if the container does
not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if (options.read_acl or options.write_acl or options.sync_to or
@@ -1529,6 +1532,8 @@ def st_post(options, args, print_queue, error_queue):
headers['X-Container-Sync-To'] = options.sync_to
if options.sync_key is not None:
headers['X-Container-Sync-Key'] = options.sync_key
+ if options.storage_policy is not None:
+ headers['X-Storage-Policy'] = options.storage_policy
try:
conn.post_container(args[0], headers=headers)
except ClientException, err:
@@ -1558,7 +1563,8 @@ upload [options] container file_or_directory
[file_or_directory] [...]
Uploads to the given container the files and directories specified by the
remaining args. -c or --changed is an option that will only upload files
that have changed since the last upload. -S <size> or --segment-size <size>
- and --leave-segments are options as well (see --help for more).
+ and --leave-segments are options as well (see --help for more).
--storage-policy
+ Sets a storage policy to the container if the container does not exist.
'''.strip('\n')
@@ -1576,6 +1582,8 @@ def st_upload(options, args, print_queue, error_queue):
dest='leave_segments', default=False, help='Indicates that you want '
'the older segments of manifest objects left alone (in the case of '
'overwrites)')
+ parser.add_option('', '--storage-policy', action='store',
dest='storage_policy',
+ help='Sets a storage policy to the container if the container does
not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if len(args) < 2:
@@ -1749,9 +1757,12 @@ def st_upload(options, args, print_queue, error_queue):
# permissions, so we'll ignore any error. If there's really a problem,
# it'll surface on the first object PUT.
try:
- conn.put_container(args[0])
+ container_headers = {}
+ if options.storage_policy is not None:
+ container_headers['X-Storage-Policy'] = options.storage_policy
+ conn.put_container(args[0],headers=container_headers)
if options.segment_size is not None:
- conn.put_container(args[0] + '_segments')
+ conn.put_container(args[0] + '_segments',headers=container_headers)
except Exception:
pass
try:
diff --git a/scripts/vm/hypervisor/xenserver/swift
b/scripts/vm/hypervisor/xenserver/swift
index c9d2ceb..603bfdc 100755
--- a/scripts/vm/hypervisor/xenserver/swift
+++ b/scripts/vm/hypervisor/xenserver/swift
@@ -1475,8 +1475,9 @@ post [options] [container] [object]
Updates meta information for the account, container, or object depending on
the args given. If the container is not found, it will be created
automatically; but this is not true for accounts and objects. Containers
- also allow the -r (or --read-acl) and -w (or --write-acl) options. The -m
- or --meta option is allowed on all and used to define the user meta data
+ also allow the -r (or --read-acl) and -w (or --write-acl) options.
+ The --storage-policy will set a storage policy to the container if the
container does not exist.
+ The -m or --meta option is allowed on all and used to define the user meta
data
items to set in the form Name:Value. This option can be repeated. Example:
post -m Color:Blue -m Size:Large'''.strip('\n')
@@ -1495,6 +1496,8 @@ def st_post(options, args, print_queue, error_queue):
parser.add_option('-m', '--meta', action='append', dest='meta', default=[],
help='Sets a meta data item with the syntax name:value. This option '
'may be repeated. Example: -m Color:Blue -m Size:Large')
+ parser.add_option('', '--storage-policy', action='store',
dest='storage_policy',
+ help='Sets a storage policy to the container if the container does
not exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if (options.read_acl or options.write_acl or options.sync_to or
@@ -1531,6 +1534,8 @@ def st_post(options, args, print_queue, error_queue):
headers['X-Container-Sync-To'] = options.sync_to
if options.sync_key is not None:
headers['X-Container-Sync-Key'] = options.sync_key
+ if options.storage_policy is not None:
+ headers['X-Storage-Policy'] = options.storage_policy
try:
conn.post_container(args[0], headers=headers)
except ClientException, err:
@@ -1560,7 +1565,8 @@ upload [options] container file_or_directory
[file_or_directory] [...]
Uploads to the given container the files and directories specified by the
remaining args. -c or --changed is an option that will only upload files
that have changed since the last upload. -S <size> or --segment-size <size>
- and --leave-segments are options as well (see --help for more).
+ and --leave-segments are options as well (see --help for more).
--storage-policy
+ Sets a storage policy to the container if the container does not exist.
'''.strip('\n')
@@ -1578,6 +1584,8 @@ def st_upload(options, args, print_queue, error_queue):
dest='leave_segments', default=False, help='Indicates that you want '
'the older segments of manifest objects left alone (in the case of '
'overwrites)')
+ parser.add_option('', '--storage-policy', action='store',
dest='storage_policy',
+ help='Sets a storage policy to the container if the container does not
exist')
(options, args) = parse_args(parser, args)
args = args[1:]
if len(args) < 2:
@@ -1751,9 +1759,12 @@ def st_upload(options, args, print_queue, error_queue):
# permissions, so we'll ignore any error. If there's really a problem,
# it'll surface on the first object PUT.
try:
- conn.put_container(args[0])
+ container_headers = {}
+ if options.storage_policy is not None:
+ container_headers['X-Storage-Policy'] = options.storage_policy
+ conn.put_container(args[0],headers=container_headers)
if options.segment_size is not None:
- conn.put_container(args[0] + '_segments')
+ conn.put_container(args[0] + '_segments',headers=container_headers)
except Exception:
pass
try:
diff --git a/ui/l10n/en.js b/ui/l10n/en.js
index 745deb1..86a8606 100644
--- a/ui/l10n/en.js
+++ b/ui/l10n/en.js
@@ -1617,6 +1617,7 @@ var dictionary = {"ICMP.code":"ICMP Code",
"label.storage.tags":"Storage Tags",
"label.storage.traffic":"Storage Traffic",
"label.storage.type":"Storage Type",
+"label.storagepolicy":"Storage policy",
"label.subdomain.access":"Subdomain Access",
"label.submit":"Submit",
"label.submitted.by":"[Submitted by: <span id=\"submitted_by\"></span>]",
diff --git a/ui/scripts/system.js b/ui/scripts/system.js
index 63f549a..d5b9976 100755
--- a/ui/scripts/system.js
+++ b/ui/scripts/system.js
@@ -19889,6 +19889,7 @@
$form.find('.form-item[rel=account]').hide();
$form.find('.form-item[rel=username]').hide();
$form.find('.form-item[rel=key]').hide();
+
$form.find('.form-item[rel=storagepolicy]').hide();
} else if
($(this).val() == "SMB") {
//NFS, SMB
$form.find('.form-item[rel=zoneid]').css('display', 'inline-block');
@@ -19921,6 +19922,7 @@
$form.find('.form-item[rel=account]').hide();
$form.find('.form-item[rel=username]').hide();
$form.find('.form-item[rel=key]').hide();
+
$form.find('.form-item[rel=storagepolicy]').hide();
} else if
($(this).val() == "S3") {
//NFS, SMB
$form.find('.form-item[rel=zoneid]').hide();
@@ -19955,6 +19957,7 @@
$form.find('.form-item[rel=account]').hide();
$form.find('.form-item[rel=username]').hide();
$form.find('.form-item[rel=key]').hide();
+
$form.find('.form-item[rel=storagepolicy]').hide();
} else if
($(this).val() == "Swift") {
//NFS, SMB
$form.find('.form-item[rel=zoneid]').hide();
@@ -19987,6 +19990,7 @@
$form.find('.form-item[rel=account]').css('display', 'inline-block');
$form.find('.form-item[rel=username]').css('display', 'inline-block');
$form.find('.form-item[rel=key]').css('display', 'inline-block');
+
$form.find('.form-item[rel=storagepolicy]').css('display', 'inline-block');
}
});
@@ -20177,14 +20181,26 @@
}
},
account: {
- label: 'label.account'
+ label: 'label.account',
+ validation: {
+ required: true
+ }
},
username: {
- label: 'label.username'
+ label: 'label.username',
+ validation: {
+ required: true
+ }
},
key: {
- label: 'label.key'
- }
+ label: 'label.key',
+ validation: {
+ required: true
+ }
+ },
+ storagepolicy: {
+ label: 'label.storagepolicy'
+ }
//Swift (end)
}
},
@@ -20351,6 +20367,11 @@
data[ 'details[' +
index.toString() + '].value'] = args.data.key;
index++;
}
+ if (args.data.storagepolicy !=
null && args.data.storagepolicy.length > 0) {
+ data[ 'details[' +
index.toString() + '].key'] = 'storagepolicy';
+ data[ 'details[' +
index.toString() + '].value'] = args.data.storagepolicy;
+ index++;
+ }
$.ajax({
url:
createURL('addImageStore'),
data: data,
diff --git a/utils/src/main/java/com/cloud/utils/SwiftUtil.java
b/utils/src/main/java/com/cloud/utils/SwiftUtil.java
index ce1bee3..34aceb5 100644
--- a/utils/src/main/java/com/cloud/utils/SwiftUtil.java
+++ b/utils/src/main/java/com/cloud/utils/SwiftUtil.java
@@ -24,26 +24,29 @@ import java.net.URL;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.SignatureException;
-import java.util.Map;
import java.util.Arrays;
import java.util.HashMap;
-import java.util.Formatter;
+import java.util.Map;
+import javax.crypto.Mac;
+import javax.crypto.spec.SecretKeySpec;
+
+import org.apache.commons.codec.binary.Hex;
import org.apache.log4j.Logger;
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.script.OutputInterpreter;
import com.cloud.utils.script.Script;
-import javax.crypto.Mac;
-import javax.crypto.spec.SecretKeySpec;
-
public class SwiftUtil {
private static Logger logger = Logger.getLogger(SwiftUtil.class);
- private static final long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L;
+ protected static final long SWIFT_MAX_SIZE = 5L * 1024L * 1024L * 1024L;
private static final String HMAC_SHA1_ALGORITHM = "HmacSHA1";
-
-
+ private static final String CD_SRC = "cd %s;";
+ private static final String SWIFT_CMD= "/usr/bin/python %s -A %s -U %s:%s
-K %s %s";
+ private static final String WITH_STORAGE_POLICY = " --storage-policy
\"%s\"";
+ private static final String WITH_SEGMENTS = " -S "+SWIFT_MAX_SIZE;
+ private static final String[] OPERATIONS_WITH_STORAGE_POLICIES =
{"post","upload"};
public interface SwiftClientCfg {
String getAccount();
@@ -53,6 +56,8 @@ public class SwiftUtil {
String getKey();
String getEndPoint();
+
+ String getStoragePolicy();
}
private static String getSwiftCLIPath() {
@@ -65,19 +70,10 @@ public class SwiftUtil {
}
public static boolean postMeta(SwiftClientCfg cfg, String container,
String object, Map<String, String> metas) {
- String swiftCli = getSwiftCLIPath();
- StringBuilder cms = new StringBuilder();
- for (Map.Entry<String, String> entry : metas.entrySet()) {
- cms.append(" -m ");
- cms.append(entry.getKey());
- cms.append(":");
- cms.append(entry.getValue());
- cms.append(" ");
- }
Script command = new Script("/bin/bash", logger);
command.add("-c");
- command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint()
+ " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() +
" post " +
- container + " " + object + " " + cms.toString());
+ command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(),"post",
container, object) + getMeta(metas));
+
OutputInterpreter.OneLineParser parser = new
OutputInterpreter.OneLineParser();
String result = command.execute(parser);
if (result != null) {
@@ -87,21 +83,14 @@ public class SwiftUtil {
}
public static String putObject(SwiftClientCfg cfg, File srcFile, String
container, String fileName) {
- String swiftCli = getSwiftCLIPath();
if (fileName == null) {
fileName = srcFile.getName();
}
- String srcDirectory = srcFile.getParent();
+
Script command = new Script("/bin/bash", logger);
- long size = srcFile.length();
command.add("-c");
- if (size <= SWIFT_MAX_SIZE) {
- command.add("cd " + srcDirectory + ";/usr/bin/python " + swiftCli
+ " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" +
cfg.getUserName() +
- " -K " + cfg.getKey() + " upload " + container + " " +
fileName);
- } else {
- command.add("cd " + srcDirectory + ";/usr/bin/python " + swiftCli
+ " -A " + cfg.getEndPoint() + " -U " + cfg.getAccount() + ":" +
cfg.getUserName() +
- " -K " + cfg.getKey() + " upload -S " + SWIFT_MAX_SIZE + " " +
container + " " + fileName);
- }
+ command.add(String.format(CD_SRC,
srcFile.getParent())+getUploadObjectCommand(cfg, getSwiftCLIPath(),
container,fileName, srcFile.length()));
+
OutputInterpreter.AllLinesParser parser = new
OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result != null) {
@@ -120,38 +109,19 @@ public class SwiftUtil {
return container + File.separator + srcFile.getName();
}
- private static StringBuilder buildSwiftCmd(SwiftClientCfg swift) {
- String swiftCli = getSwiftCLIPath();
- StringBuilder sb = new StringBuilder();
- sb.append(" /usr/bin/python ");
- sb.append(swiftCli);
- sb.append(" -A ");
- sb.append(swift.getEndPoint());
- sb.append(" -U ");
- sb.append(swift.getAccount());
- sb.append(":");
- sb.append(swift.getUserName());
- sb.append(" -K ");
- sb.append(swift.getKey());
- sb.append(" ");
- return sb;
- }
-
public static String[] list(SwiftClientCfg swift, String container, String
rFilename) {
- getSwiftCLIPath();
- Script command = new Script("/bin/bash", logger);
- command.add("-c");
-
- StringBuilder swiftCmdBuilder = buildSwiftCmd(swift);
- swiftCmdBuilder.append(" list ");
- swiftCmdBuilder.append(container);
+ StringBuilder swiftCmdBuilder = new StringBuilder();
+ swiftCmdBuilder.append(getSwiftContainerCmd(swift, getSwiftCLIPath(),
"list", container));
if (rFilename != null) {
swiftCmdBuilder.append(" -p ");
swiftCmdBuilder.append(rFilename);
}
+ Script command = new Script("/bin/bash", logger);
+ command.add("-c");
command.add(swiftCmdBuilder.toString());
+
OutputInterpreter.AllLinesParser parser = new
OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result == null && parser.getLines() != null &&
!parser.getLines().equalsIgnoreCase("")) {
@@ -178,11 +148,11 @@ public class SwiftUtil {
} else {
destFilePath = destDirectory.getAbsolutePath();
}
- String swiftCli = getSwiftCLIPath();
+
Script command = new Script("/bin/bash", logger);
command.add("-c");
- command.add("/usr/bin/python " + swiftCli + " -A " + cfg.getEndPoint()
+ " -U " + cfg.getAccount() + ":" + cfg.getUserName() + " -K " + cfg.getKey() +
- " download " + container + " " + srcPath + " -o " +
destFilePath);
+ command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(), "download",
container, srcPath)+" -o " + destFilePath);
+
OutputInterpreter.AllLinesParser parser = new
OutputInterpreter.AllLinesParser();
String result = command.execute(parser);
if (result != null) {
@@ -203,27 +173,6 @@ public class SwiftUtil {
return new File(destFilePath);
}
- public static String getContainerName(String type, Long id) {
- if (type.startsWith("T")) {
- return "T-" + id;
- } else if (type.startsWith("S")) {
- return "S-" + id;
- } else if (type.startsWith("V")) {
- return "V-" + id;
- }
- return null;
- }
-
- public static String[] splitSwiftPath(String path) {
- int index = path.indexOf(File.separator);
- if (index == -1) {
- return null;
- }
- String[] paths = new String[2];
- paths[0] = path.substring(0, index);
- paths[1] = path.substring(index + 1);
- return paths;
- }
public static boolean deleteObject(SwiftClientCfg cfg, String path) {
Script command = new Script("/bin/bash", logger);
@@ -236,13 +185,8 @@ public class SwiftUtil {
String container = paths[0];
String objectName = paths[1];
- StringBuilder swiftCmdBuilder = buildSwiftCmd(cfg);
- swiftCmdBuilder.append(" delete ");
- swiftCmdBuilder.append(container);
- swiftCmdBuilder.append(" ");
- swiftCmdBuilder.append(objectName);
+ command.add(getSwiftObjectCmd(cfg, getSwiftCLIPath(), "delete",
container, objectName));
- command.add(swiftCmdBuilder.toString());
OutputInterpreter.AllLinesParser parser = new
OutputInterpreter.AllLinesParser();
command.execute(parser);
return true;
@@ -284,7 +228,7 @@ public class SwiftUtil {
}
- public static String calculateRFC2104HMAC(String data, String key)
+ static String calculateRFC2104HMAC(String data, String key)
throws SignatureException, NoSuchAlgorithmException,
InvalidKeyException {
SecretKeySpec signingKey = new SecretKeySpec(key.getBytes(),
HMAC_SHA1_ALGORITHM);
@@ -294,12 +238,75 @@ public class SwiftUtil {
}
- public static String toHexString(byte[] bytes) {
+ static String toHexString(byte[] bytes) {
+ return Hex.encodeHexString(bytes);
+ }
+
+ /////////////// SWIFT CMD STRING HELPERS ///////////////
+ protected static String getSwiftCmd(SwiftClientCfg cfg, String swiftCli,
String operation){
+ return String.format(SWIFT_CMD,
swiftCli,cfg.getEndPoint(),cfg.getAccount(),cfg.getUserName(),cfg.getKey(),operation);
+ }
+
+ protected static String getSwiftObjectCmd(SwiftClientCfg cfg, String
swiftCliPath, String operation,String container, String objectName) {
+ String cmd = getSwiftCmd(cfg,swiftCliPath, operation) +" "+
container+" "+objectName;
+ if(StringUtils.isNotBlank(cfg.getStoragePolicy()) &&
supportsStoragePolicies(operation)){
+ return cmd + String.format(WITH_STORAGE_POLICY,
cfg.getStoragePolicy());
+ }
+ return cmd;
+ }
+
+ private static boolean supportsStoragePolicies(String operation) {
+ for(String supportedOp: OPERATIONS_WITH_STORAGE_POLICIES){
+ if(supportedOp.equals(operation)){
+ return true;
+ }
+ }
+ return false;
+ }
+
+ protected static String getSwiftContainerCmd(SwiftClientCfg cfg, String
swiftCliPath, String operation, String container) {
+ return getSwiftCmd(cfg,swiftCliPath, operation) +" "+ container;
+ }
+
+ protected static String getUploadObjectCommand(SwiftClientCfg cfg, String
swiftCliPath, String container, String objectName, long size) {
+ String cmd = getSwiftObjectCmd(cfg, swiftCliPath, "upload", container,
objectName);
+ if(size > SWIFT_MAX_SIZE){
+ return cmd + WITH_SEGMENTS;
+ }
+ return cmd;
+ }
+
+ public static String getContainerName(String type, Long id) {
+ if (type.startsWith("T")) {
+ return "T-" + id;
+ } else if (type.startsWith("S")) {
+ return "S-" + id;
+ } else if (type.startsWith("V")) {
+ return "V-" + id;
+ }
+ return null;
+ }
+
+ public static String[] splitSwiftPath(String path) {
+ int index = path.indexOf(File.separator);
+ if (index == -1) {
+ return null;
+ }
+ String[] paths = new String[2];
+ paths[0] = path.substring(0, index);
+ paths[1] = path.substring(index + 1);
+ return paths;
+ }
- Formatter formatter = new Formatter();
- for (byte b : bytes) {
- formatter.format("%02x", b);
+ private static String getMeta(Map<String, String> metas) {
+ StringBuilder cms = new StringBuilder();
+ for (Map.Entry<String, String> entry : metas.entrySet()) {
+ cms.append(" -m ");
+ cms.append(entry.getKey());
+ cms.append(":");
+ cms.append(entry.getValue());
+ cms.append(" ");
}
- return formatter.toString();
+ return cms.toString();
}
-}
+}
\ No newline at end of file
diff --git a/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java
b/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java
index 20c1623..6dc2cc7 100644
--- a/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java
+++ b/utils/src/test/java/com/cloud/utils/SwiftUtilTest.java
@@ -19,9 +19,15 @@
package com.cloud.utils;
-
-import org.junit.Test;
-import org.mockito.Mockito;
+import static org.mockito.BDDMockito.given;
+import static org.mockito.BDDMockito.mock;
+import static org.junit.Assert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.when;
import java.io.File;
import java.net.URL;
@@ -29,10 +35,10 @@ import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.security.SignatureException;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.when;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import com.cloud.utils.SwiftUtil.SwiftClientCfg;
public class SwiftUtilTest {
@@ -90,4 +96,134 @@ public class SwiftUtilTest {
assertEquals(expected, output);
}
+
+ @Test
+ public void testGetSwiftCmd() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String cmd = SwiftUtil.getSwiftCmd(cfg, "swift", "stat");
+
+ String expected = "/usr/bin/python swift -A swift.endpoint -U
cs:sec-storage -K mypassword stat";
+ assertThat(cmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetSwiftObjectCmd() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String objectCmd = SwiftUtil.getSwiftObjectCmd(cfg, "swift", "delete",
"T-123", "template.vhd");
+
+ String expected = "/usr/bin/python swift -A swift.endpoint -U
cs:sec-storage -K mypassword delete T-123 template.vhd";
+ assertThat(objectCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetSwiftContainerCmd() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String containerCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift",
"list", "T-123");
+
+ String expected = "/usr/bin/python swift -A swift.endpoint -U
cs:sec-storage -K mypassword list T-123";
+ assertThat(containerCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetUploadCmd() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift",
"T-1", "template.vhd", 1024);
+
+ String expected = "/usr/bin/python swift -A swift.endpoint -U
cs:sec-storage -K mypassword upload T-1 template.vhd";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetUploadCmdWithSegmentsBecauseOfSize() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift",
"T-1", "template.vhd", 5368709121L);
+
+ String expected = "/usr/bin/python swift -A swift.endpoint -U
cs:sec-storage -K mypassword upload T-1 template.vhd -S 5368709120";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetUploadCmdWithStoragePolicy() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn("policy1");
+
+ String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift",
"T-1", "template.vhd", 1024L);
+ String expected = "/usr/bin/python swift -A swift.endpoint -U
cs:sec-storage -K mypassword upload T-1 template.vhd --storage-policy
\"policy1\"";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testGetUploadCmdWithSegmentsAndStoragePolicy() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn("policy1");
+ String uploadCmd = SwiftUtil.getUploadObjectCommand(cfg, "swift",
"T-1", "template.vhd", 5368709121L);
+ String expected = "/usr/bin/python swift -A swift.endpoint -U
cs:sec-storage -K mypassword upload T-1 template.vhd --storage-policy
\"policy1\" -S 5368709120";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void
testListContainerCmdWithStoragePolicyButNotSupportedByOperation() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn("policy1");
+
+ String uploadCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift",
"list", "T-1");
+ String expected = "/usr/bin/python swift -A swift.endpoint -U
cs:sec-storage -K mypassword list T-1";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
+
+ @Test
+ public void testListContainerCmdWithoutStoragePolicy() {
+ SwiftClientCfg cfg = mock(SwiftClientCfg.class);
+ given(cfg.getEndPoint()).willReturn("swift.endpoint");
+ given(cfg.getAccount()).willReturn("cs");
+ given(cfg.getUserName()).willReturn("sec-storage");
+ given(cfg.getKey()).willReturn("mypassword");
+ given(cfg.getStoragePolicy()).willReturn(null);
+
+ String uploadCmd = SwiftUtil.getSwiftContainerCmd(cfg, "swift",
"list", "T-1");
+ String expected = "/usr/bin/python swift -A swift.endpoint -U
cs:sec-storage -K mypassword list T-1";
+ assertThat(uploadCmd, is(equalTo(expected)));
+ }
}
--
To stop receiving notification emails like this one, please contact
[email protected].
