http://git-wip-us.apache.org/repos/asf/cloudstack/blob/bf32776f/server/test/com/cloud/upgrade/Usage217To224UpgradeTest.java ---------------------------------------------------------------------- diff --git a/server/test/com/cloud/upgrade/Usage217To224UpgradeTest.java b/server/test/com/cloud/upgrade/Usage217To224UpgradeTest.java deleted file mode 100644 index 741af5a..0000000 --- a/server/test/com/cloud/upgrade/Usage217To224UpgradeTest.java +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.upgrade; - - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; - -import javax.inject.Inject; - -import junit.framework.TestCase; - -import org.apache.log4j.Logger; -import org.junit.After; -import org.junit.Before; - -import com.cloud.upgrade.dao.VersionDaoImpl; - -import com.cloud.utils.db.DbTestUtils; -import com.cloud.utils.db.Transaction; - -public class Usage217To224UpgradeTest extends TestCase { - private static final Logger s_logger = Logger.getLogger(Usage217To224UpgradeTest.class); - - @Inject VersionDaoImpl dao; - @Inject PremiumDatabaseUpgradeChecker checker; - - @Override - @Before - public void setUp() throws Exception { - DbTestUtils.executeScript("cleanup.sql", false, true); - DbTestUtils.executeUsageScript("cleanup.sql", false, true); - } - - @Override - @After - public void tearDown() throws Exception { - } - - public void test21to22Upgrade() throws SQLException { - s_logger.debug("Finding sample data from 2.1.7"); - DbTestUtils.executeScript("fake.sql", false, true); - DbTestUtils.executeUsageScript("fake.sql", false, true); - - Connection conn; - PreparedStatement pstmt; - - - String version = dao.getCurrentVersion(); - assert version.equals("2.1.7") : "Version returned is not 2.1.7 but " + version; - - checker.upgrade("2.1.7", "2.2.4"); - - conn = Transaction.getStandaloneConnection(); - try { - pstmt = conn.prepareStatement("SELECT version FROM version ORDER BY id DESC LIMIT 1"); - ResultSet rs = pstmt.executeQuery(); - assert rs.next() : "No version selected"; - assert rs.getString(1).equals("2.2.4") : "VERSION stored is not 2.2.4: " + rs.getString(1); - rs.close(); - pstmt.close(); - - pstmt = conn.prepareStatement("SELECT COUNT(*) FROM usage_event"); - rs = pstmt.executeQuery(); - assert rs.next() : "Unable to get the count of usage events"; - assert (rs.getInt(1) == 182) : "Didn't find 182 usage events but found " + rs.getInt(1); - rs.close(); - pstmt.close(); - - } finally { - try { - conn.close(); - } catch (SQLException e) { - } - } - } - -}
http://git-wip-us.apache.org/repos/asf/cloudstack/blob/bf32776f/server/test/com/cloud/upgrade/UsageEvents218To224UpgradeTest.java ---------------------------------------------------------------------- diff --git a/server/test/com/cloud/upgrade/UsageEvents218To224UpgradeTest.java b/server/test/com/cloud/upgrade/UsageEvents218To224UpgradeTest.java deleted file mode 100644 index cde114b..0000000 --- a/server/test/com/cloud/upgrade/UsageEvents218To224UpgradeTest.java +++ /dev/null @@ -1,91 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. -package com.cloud.upgrade; - - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; - -import javax.inject.Inject; - -import junit.framework.TestCase; - -import org.apache.log4j.Logger; -import org.junit.After; -import org.junit.Before; - -import com.cloud.upgrade.dao.VersionDaoImpl; - -import com.cloud.utils.db.DbTestUtils; -import com.cloud.utils.db.Transaction; - -public class UsageEvents218To224UpgradeTest extends TestCase { - private static final Logger s_logger = Logger.getLogger(UsageEvents218To224UpgradeTest.class); - - @Inject VersionDaoImpl dao; - @Inject DatabaseUpgradeChecker checker; - - @Override - @Before - public void setUp() throws Exception { - DbTestUtils.executeScript("cleanup.sql", false, true); - } - - @Override - @After - public void tearDown() throws Exception { - } - - public void test21to22Upgrade() throws SQLException { - s_logger.debug("Finding sample data from 2.1.8"); - DbTestUtils.executeScript("fake.sql", false, true); - - Connection conn; - PreparedStatement pstmt; - - String version = dao.getCurrentVersion(); - assert version.equals("2.1.8") : "Version returned is not 2.1.8 but " + version; - - checker.upgrade("2.1.8", "2.2.4"); - - conn = Transaction.getStandaloneConnection(); - try { - pstmt = conn.prepareStatement("SELECT version FROM version ORDER BY id DESC LIMIT 1"); - ResultSet rs = pstmt.executeQuery(); - assert rs.next() : "No version selected"; - assert rs.getString(1).equals("2.2.4") : "VERSION stored is not 2.2.4: " + rs.getString(1); - rs.close(); - pstmt.close(); - - pstmt = conn.prepareStatement("SELECT COUNT(*) FROM usage_event"); - rs = pstmt.executeQuery(); - assert rs.next() : "Unable to get the count of usage events"; - assert (rs.getInt(1) == 37) : "Didn't find 37 usage events but found " + rs.getInt(1); - rs.close(); - pstmt.close(); - - } finally { - try { - conn.close(); - } catch (SQLException e) { - } - } - } - -} http://git-wip-us.apache.org/repos/asf/cloudstack/blob/bf32776f/services/secondary-storage/pom.xml ---------------------------------------------------------------------- diff --git a/services/secondary-storage/pom.xml b/services/secondary-storage/pom.xml index 124fa5e..7d2089b 100644 --- a/services/secondary-storage/pom.xml +++ b/services/secondary-storage/pom.xml @@ -66,6 +66,14 @@ <testSourceDirectory>test</testSourceDirectory> <plugins> <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <version>2.14</version> + <configuration> + <skipTests>true</skipTests> + </configuration> + </plugin> + <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>exec-maven-plugin</artifactId> <version>1.2.1</version> http://git-wip-us.apache.org/repos/asf/cloudstack/blob/bf32776f/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java ---------------------------------------------------------------------- diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java index 1a6dcf8..fd6e296 100644 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResource.java @@ -65,13 +65,10 @@ public class LocalNfsSecondaryStorageResource extends NfsSecondaryStorageResourc ((DownloadManagerImpl) _dlMgr).setThreadPool(Executors.newFixedThreadPool(10)); _storage = new JavaStorageLayer(); this._inSystemVM = false; - // get mount parent folder configured in global setting, if set, this will overwrite _parent in NfsSecondaryStorageResource to work - // around permission issue for default /mnt folder - ConfigurationDaoImpl configDao = new ConfigurationDaoImpl(); - String mountParent = configDao.getValue(Config.MountParent.key()); - if (mountParent != null) { - _parent = mountParent + File.separator + "secStorage"; - } + } + + public void setParentPath(String path) { + this._parent = path; } @Override http://git-wip-us.apache.org/repos/asf/cloudstack/blob/bf32776f/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java ---------------------------------------------------------------------- diff --git a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java index 88e3784..054edd9 100755 --- a/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java +++ b/services/secondary-storage/src/org/apache/cloudstack/storage/resource/NfsSecondaryStorageResource.java @@ -536,7 +536,7 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } } - protected File downloadFromUrlToNfs(String url, NfsTO nfs, String path) { + protected File downloadFromUrlToNfs(String url, NfsTO nfs, String path, String name) { HttpClient client = new DefaultHttpClient(); HttpGet get = new HttpGet(url); try { @@ -548,10 +548,19 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S } String nfsMountPath = getRootDir(nfs.getUrl()); + String filePath = nfsMountPath + File.separator + path; - FileOutputStream outputStream = new FileOutputStream(filePath); + File directory = new File(filePath); + if (!directory.exists()) { + _storage.mkdirs(filePath); + } + File destFile = new File(filePath + File.separator + name); + if (!destFile.exists()) { + destFile.createNewFile(); + } + FileOutputStream outputStream = new FileOutputStream(destFile); entity.writeTo(outputStream); - return new File(filePath); + return new File(destFile.getAbsolutePath()); } catch (IOException e) { s_logger.debug("Faild to get url:"+ url + ", due to " + e.toString()); throw new CloudRuntimeException(e); @@ -565,9 +574,11 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S return new DownloadAnswer("cache store can't be null", VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); } + File file = null; try { NfsTO nfsCacheStore = (NfsTO)cacheStore; - File file = downloadFromUrlToNfs(cmd.getUrl(), nfsCacheStore, path); + String fileName = UUID.randomUUID().toString() + "." + cmd.getFormat().getFileExtension(); + file = downloadFromUrlToNfs(cmd.getUrl(), nfsCacheStore, path, fileName); String swiftPath = SwiftUtil.putObject(swiftTO, file, "T-" + cmd.getId()); String md5sum = null; try { @@ -576,14 +587,17 @@ public class NfsSecondaryStorageResource extends ServerResourceBase implements S s_logger.debug("Failed to get md5sum: " + file.getAbsoluteFile()); } - file.delete(); - - return new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, + DownloadAnswer answer = new DownloadAnswer(null, 100, null, VMTemplateStorageResourceAssoc.Status.DOWNLOADED, swiftPath, swiftPath, file.length(), file.length(), md5sum ); + return answer; } catch (Exception e) { s_logger.debug("Failed to register template into swift", e); return new DownloadAnswer(e.toString(), VMTemplateStorageResourceAssoc.Status.DOWNLOAD_ERROR); + } finally { + if (file != null) { + file.delete(); + } } } http://git-wip-us.apache.org/repos/asf/cloudstack/blob/bf32776f/services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java ---------------------------------------------------------------------- diff --git a/services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java b/services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java new file mode 100644 index 0000000..7723321 --- /dev/null +++ b/services/secondary-storage/test/org/apache/cloudstack/storage/resource/LocalNfsSecondaryStorageResourceTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.cloudstack.storage.resource; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.storage.DownloadAnswer; +import com.cloud.agent.api.to.DataObjectType; +import com.cloud.agent.api.to.NfsTO; +import com.cloud.agent.api.to.SwiftTO; +import com.cloud.storage.DataStoreRole; +import com.cloud.storage.Storage; +import com.cloud.utils.SwiftUtil; +import junit.framework.Assert; +import junit.framework.TestCase; +import org.apache.cloudstack.storage.command.CopyCmdAnswer; +import org.apache.cloudstack.storage.command.CopyCommand; +import org.apache.cloudstack.storage.command.DownloadCommand; +import org.apache.cloudstack.storage.to.TemplateObjectTO; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; + + +import javax.naming.ConfigurationException; +import java.util.HashMap; + +public class LocalNfsSecondaryStorageResourceTest extends TestCase { + LocalNfsSecondaryStorageResource resource; + @Before + @Override + public void setUp() throws ConfigurationException { + resource = new LocalNfsSecondaryStorageResource(); + resource.setParentPath("/mnt"); + System.setProperty("paths.script", "/Users/edison/develop/asf-master/script"); + //resource.configure("test", new HashMap<String, Object>()); + } + @Test + public void testExecuteRequest() throws Exception { + TemplateObjectTO template = Mockito.mock(TemplateObjectTO.class); + NfsTO cacheStore = Mockito.mock(NfsTO.class); + Mockito.when(cacheStore.getUrl()).thenReturn("nfs://nfs2.lab.vmops.com/export/home/edison/"); + SwiftTO swift = Mockito.mock(SwiftTO.class); + Mockito.when(swift.getEndPoint()).thenReturn("https://objects.dreamhost.com/auth";); + Mockito.when(swift.getAccount()).thenReturn("cloudstack"); + Mockito.when(swift.getUserName()).thenReturn("images"); + //Mockito.when(swift.getKey()).thenReturn("something"); + + Mockito.when(template.getDataStore()).thenReturn(swift); + Mockito.when(template.getPath()).thenReturn("template/1/1/"); + Mockito.when(template.isRequiresHvm()).thenReturn(true); + Mockito.when(template.getId()).thenReturn(1L); + Mockito.when(template.getFormat()).thenReturn(Storage.ImageFormat.VHD); + Mockito.when(template.getOrigUrl()).thenReturn("http://nfs1.lab.vmops.com/templates/ttylinux_pv.vhd";); + Mockito.when(template.getObjectType()).thenReturn(DataObjectType.TEMPLATE); + + DownloadCommand cmd = new DownloadCommand(template, 100000L); + cmd.setCacheStore(cacheStore); + DownloadAnswer answer = (DownloadAnswer)resource.executeRequest(cmd); + Assert.assertTrue(answer.getResult()); + + Mockito.when(template.getPath()).thenReturn(answer.getInstallPath()); + Mockito.when(template.getDataStore()).thenReturn(swift); + //download swift: + Mockito.when(cacheStore.getRole()).thenReturn(DataStoreRole.ImageCache); + TemplateObjectTO destTemplate = Mockito.mock(TemplateObjectTO.class); + Mockito.when(destTemplate.getPath()).thenReturn("template/1/2"); + Mockito.when(destTemplate.getDataStore()).thenReturn(cacheStore); + Mockito.when(destTemplate.getObjectType()).thenReturn(DataObjectType.TEMPLATE); + CopyCommand cpyCmd = new CopyCommand(template, destTemplate, 10000, true); + CopyCmdAnswer copyCmdAnswer = (CopyCmdAnswer)resource.executeRequest(cpyCmd); + Assert.assertTrue(copyCmdAnswer.getResult()); + + } +} http://git-wip-us.apache.org/repos/asf/cloudstack/blob/bf32776f/tools/marvin/marvin/deployDataCenter.py ---------------------------------------------------------------------- diff --git a/tools/marvin/marvin/deployDataCenter.py b/tools/marvin/marvin/deployDataCenter.py index 20908ca..100f442 100644 --- a/tools/marvin/marvin/deployDataCenter.py +++ b/tools/marvin/marvin/deployDataCenter.py @@ -159,7 +159,7 @@ specify a valid config file" % cfgFile) secondarycmd.provider = secondary.provider secondarycmd.details = [] - if secondarycmd.provider == 'S3': + if secondarycmd.provider == 'S3' or secondarycmd.provider == "Swift": for key, value in vars(secondary.details).iteritems(): secondarycmd.details.append({ 'key': key, http://git-wip-us.apache.org/repos/asf/cloudstack/blob/bf32776f/utils/src/com/cloud/utils/SwiftUtil.java ---------------------------------------------------------------------- diff --git a/utils/src/com/cloud/utils/SwiftUtil.java b/utils/src/com/cloud/utils/SwiftUtil.java index b95249b..c01de86 100644 --- a/utils/src/com/cloud/utils/SwiftUtil.java +++ b/utils/src/com/cloud/utils/SwiftUtil.java @@ -50,6 +50,7 @@ public class SwiftUtil { String srcDirectory = srcFile.getParent(); Script command = new Script("/bin/bash", logger); long size = srcFile.length(); + command.add("-c"); if (size <= SWIFT_MAX_SIZE) { command.add("cd " + srcDirectory + ";/usr/bin/python " + swiftCli + " -A "
