[
https://issues.apache.org/jira/browse/CLOUDSTACK-2583?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=13703092#comment-13703092
]
Sanjeev N commented on CLOUDSTACK-2583:
---------------------------------------
Verified this issue on the latest master build. This time I din't see any
issues with s3xen persent on xenserver.
Snapshot creation was successful, however copying to s3 image store failed.
Following is the exception from the management server log file:
2013-07-09 10:53:01,518 DEBUG [cloud.async.AsyncJobManagerImpl]
(Job-Executor-10:job-10) Executing
org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotCmd for job-10
2013-07-09 10:53:01,582 INFO [user.snapshot.CreateSnapshotCmd]
(Job-Executor-10:job-10) VOLSS: createSnapshotCmd starts:1373381581582
2013-07-09 10:53:01,685 DEBUG [agent.transport.Request]
(Job-Executor-10:job-10) Seq 1-1762853387: Sending { Cmd , MgmtId:
6615759585382, via: 1, Ver: v1, Flags: 100011,
[{"org.apache.cloudstack.storage.command.CreateObjectCommand":{"data":{"org.apache.cloudstack.storage.to.SnapshotObjectTO":{"volume":{"uuid":"97e02ab2-c42f-42cc-816a-570dac81d6c8","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"c65a038a-750c-3b4f-bf26-7ce3b74e1c85","id":1,"poolType":"NetworkFilesystem","host":"10.147.28.7","path":"/export/home/sanjeev/pri_xen_os","port":2049}},"name":"ROOT-4","size":21474836480,"path":"28677700-7303-4015-a386-e55fb0f0ac89","volumeId":4,"vmName":"i-2-4-VM","accountId":2,"format":"VHD","id":4},"dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"c65a038a-750c-3b4f-bf26-7ce3b74e1c85","id":1,"poolType":"NetworkFilesystem","host":"10.147.28.7","path":"/export/home/sanjeev/pri_xen_os","port":2049}},"vmName":"i-2-4-VM","name":"vm1_ROOT-4_20130709145301","hypervisorType":"XenServer","id":2}},"wait":0}}]
}
2013-07-09 10:53:01,687 DEBUG [agent.transport.Request]
(Job-Executor-10:job-10) Seq 1-1762853387: Executing: { Cmd , MgmtId:
6615759585382, via: 1, Ver: v1, Flags: 100011,
[{"org.apache.cloudstack.storage.command.CreateObjectCommand":{"data":{"org.apache.cloudstack.storage.to.SnapshotObjectTO":{"volume":{"uuid":"97e02ab2-c42f-42cc-816a-570dac81d6c8","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"c65a038a-750c-3b4f-bf26-7ce3b74e1c85","id":1,"poolType":"NetworkFilesystem","host":"10.147.28.7","path":"/export/home/sanjeev/pri_xen_os","port":2049}},"name":"ROOT-4","size":21474836480,"path":"28677700-7303-4015-a386-e55fb0f0ac89","volumeId":4,"vmName":"i-2-4-VM","accountId":2,"format":"VHD","id":4},"dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"c65a038a-750c-3b4f-bf26-7ce3b74e1c85","id":1,"poolType":"NetworkFilesystem","host":"10.147.28.7","path":"/export/home/sanjeev/pri_xen_os","port":2049}},"vmName":"i-2-4-VM","name":"vm1_ROOT-4_20130709145301","hypervisorType":"XenServer","id":2}},"wait":0}}]
}
2013-07-09 10:53:01,687 DEBUG [agent.manager.DirectAgentAttache]
(DirectAgent-163:null) Seq 1-1762853387: Executing request
2013-07-09 10:53:03,029 DEBUG [agent.manager.DirectAgentAttache]
(DirectAgent-163:null) Seq 1-1762853387: Response Received:
2013-07-09 10:53:03,030 DEBUG [agent.transport.Request] (DirectAgent-163:null)
Seq 1-1762853387: Processing: { Ans: , MgmtId: 6615759585382, via: 1, Ver: v1,
Flags: 10,
[{"org.apache.cloudstack.storage.command.CreateObjectAnswer":{"data":{"org.apache.cloudstack.storage.to.SnapshotObjectTO":{"path":"a5b44a59-f4d1-4ea1-ba76-5513cb595daa","id":0}},"result":true,"wait":0}}]
}
2013-07-09 10:53:03,030 DEBUG [agent.transport.Request]
(Job-Executor-10:job-10) Seq 1-1762853387: Received: { Ans: , MgmtId:
6615759585382, via: 1, Ver: v1, Flags: 10, { CreateObjectAnswer } }
2013-07-09 10:53:03,154 DEBUG [storage.motion.AncientDataMotionStrategy]
(Job-Executor-10:job-10) copyAsync inspecting src type SNAPSHOT copyAsync
inspecting dest type SNAPSHOT
2013-07-09 10:53:03,201 DEBUG [storage.motion.AncientDataMotionStrategy]
(Job-Executor-10:job-10) needCacheStorage true, dest at snapshots/2/4 dest role
Imagea5b44a59-f4d1-4ea1-ba76-5513cb595daa src role Primary
2013-07-09 10:53:03,219 DEBUG [storage.motion.AncientDataMotionStrategy]
(Job-Executor-10:job-10) copy snasphot failed: java.lang.NullPointerException
2013-07-09 10:53:03,219 DEBUG [storage.motion.AncientDataMotionStrategy]
(Job-Executor-10:job-10) copy failed
com.cloud.utils.exception.CloudRuntimeException: java.lang.NullPointerException
at
org.apache.cloudstack.storage.motion.AncientDataMotionStrategy.copySnapshot(AncientDataMotionStrategy.java:417)
at
org.apache.cloudstack.storage.motion.AncientDataMotionStrategy.copyAsync(AncientDataMotionStrategy.java:349)
at
org.apache.cloudstack.storage.motion.DataMotionServiceImpl.copyAsync(DataMotionServiceImpl.java:55)
at
org.apache.cloudstack.storage.snapshot.SnapshotServiceImpl.backupSnapshot(SnapshotServiceImpl.java:267)
at
org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.backupSnapshot(XenserverSnapshotStrategy.java:127)
at
org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.takeSnapshot(XenserverSnapshotStrategy.java:197)
at
com.cloud.storage.snapshot.SnapshotManagerImpl.takeSnapshot(SnapshotManagerImpl.java:981)
at
org.apache.cloudstack.storage.volume.VolumeServiceImpl.takeSnapshot(VolumeServiceImpl.java:1171)
at
com.cloud.storage.VolumeManagerImpl.takeSnapshot(VolumeManagerImpl.java:2663)
at
org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotCmd.execute(CreateSnapshotCmd.java:170)
at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:155)
at
com.cloud.async.AsyncJobManagerImpl$1.run(AsyncJobManagerImpl.java:514)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
2013-07-09 10:53:03,240 DEBUG [storage.snapshot.SnapshotManagerImpl]
(Job-Executor-10:job-10) Failed to create snapshot
com.cloud.utils.exception.CloudRuntimeException:
com.cloud.utils.exception.CloudRuntimeException: java.lang.NullPointerException
at
org.apache.cloudstack.storage.snapshot.SnapshotServiceImpl.backupSnapshot(SnapshotServiceImpl.java:282)
at
org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.backupSnapshot(XenserverSnapshotStrategy.java:127)
at
org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.takeSnapshot(XenserverSnapshotStrategy.java:197)
at
com.cloud.storage.snapshot.SnapshotManagerImpl.takeSnapshot(SnapshotManagerImpl.java:981)
at
org.apache.cloudstack.storage.volume.VolumeServiceImpl.takeSnapshot(VolumeServiceImpl.java:1171)
at
com.cloud.storage.VolumeManagerImpl.takeSnapshot(VolumeManagerImpl.java:2663)
at
org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotCmd.execute(CreateSnapshotCmd.java:170)
at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:155)
at
com.cloud.async.AsyncJobManagerImpl$1.run(AsyncJobManagerImpl.java:514)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603)
at java.lang.Thread.run(Thread.java:679)
2013-07-09 10:53:03,259 DEBUG [storage.volume.VolumeServiceImpl]
(Job-Executor-10:job-10) Take snapshot: 4 failed:
com.cloud.utils.exception.CloudRuntimeException: Failed to create snapshot
2013-07-09 10:53:03,270 DEBUG [cloud.async.AsyncJobManagerImpl]
(Job-Executor-10:job-10) Complete async job-10, jobStatus: 2, resultCode: 530,
result: Error Code: 530 Error text: Failed to create snapshot due to an
internal error creating snapshot for volume 4
I could see the snapshot file 28677700-7303-4015-a386-e55fb0f0ac89.vhd on my
primary storage.
mysql> select * from snapshots;
+----+----------------+------------+-----------+-----------+------------------+------------------+------+---------------------------+--------------------------------------+---------------+------------------+-------------+---------------------+---------+----------------+----------+------------+--------------+-----------------+---------+-------+
| id | data_center_id | account_id | domain_id | volume_id | disk_offering_id |
status | path | name | uuid
| snapshot_type | type_description | size | created
| removed | backup_snap_id | swift_id | sechost_id | prev_snap_id |
hypervisor_type | version | s3_id |
+----+----------------+------------+-----------+-----------+------------------+------------------+------+---------------------------+--------------------------------------+---------------+------------------+-------------+---------------------+---------+----------------+----------+------------+--------------+-----------------+---------+-------+
| 1 | 1 | 2 | 1 | 4 | 1 |
CreatedOnPrimary | NULL | vm1_ROOT-4_20130709144517 |
4e435bc7-9b10-49e9-a1a7-711d29537df3 | 0 | MANUAL |
21474836480 | 2013-07-09 14:45:17 | NULL | NULL | NULL |
NULL | NULL | XenServer | 2.2 | NULL |
| 2 | 1 | 2 | 1 | 4 | 1 |
CreatedOnPrimary | NULL | vm1_ROOT-4_20130709145301 |
9bd6a928-c409-4ca5-841e-8e4be510fdda | 0 | MANUAL |
21474836480 | 2013-07-09 14:53:01 | NULL | NULL | NULL |
NULL | NULL | XenServer | 2.2 | NULL |
+----+----------------+------------+-----------+-----------+------------------+------------------+------+---------------------------+--------------------------------------+---------------+------------------+-------------+---------------------+---------+----------------+----------+------------+--------------+-----------------+---------+-------+
2 rows in set (0.01 sec)
Also I dint see any erros/failures in SMLog in xenserver during snapshot
creation.
> [Object_Store_Refactor] Failed to create snapshot from root disk of a guest VM
> ------------------------------------------------------------------------------
>
> Key: CLOUDSTACK-2583
> URL: https://issues.apache.org/jira/browse/CLOUDSTACK-2583
> Project: CloudStack
> Issue Type: Bug
> Security Level: Public(Anyone can view this level - this is the
> default.)
> Components: Snapshot
> Affects Versions: 4.2.0
> Environment: Build from object_store feature branch
> Reporter: Sanjeev N
> Assignee: John Burwell
> Priority: Blocker
> Fix For: 4.2.0
>
> Attachments: management-server.rar
>
>
> [Object_Store_Refactor] Failed to create snapshot from root disk of a guest VM
> Steps to Reproduce:
> =================
> 1.Bring up CS in advanced zone with S3 as the secondary storage provider
> 2.Deploy guest with with the default centos template template downloaded
> 3.Try to take snapshot on the root disk of the guest vm
> Observations:
> ======2013-05-20 11:51:21,930 DEBUG [cloud.api.ApiServlet]
> (catalina-exec-13:null) ===START=== 10.146.0.15 -- GET
> command=createSnapshot&volumeid=135a3df1-20e1-4b4b-a433-86976ddc9218&response=json&sessionkey=YUyQ2NNnnp%2FGwC0zjw71QxChfHQ%3D&_=1369045371277
> 2013-05-20 11:51:22,046 DEBUG [agent.manager.DirectAgentAttache]
> (DirectAgent-281:null) Seq 1-774570942: Response Received:
> 2013-05-20 11:51:22,046 DEBUG [agent.transport.Request]
> (StatsCollector-2:null) Seq 1-774570942: Received: { Ans: , MgmtId:
> 7332683579487, via: 1, Ver: v1, Flags: 10, { GetStorageStatsAnswer } }
> 2013-05-20 11:51:22,124 DEBUG [cloud.async.AsyncJobManagerImpl]
> (catalina-exec-13:null) submit async job-17, details: AsyncJobVO {id:17,
> userId: 2, accountId: 2, sessionKey: null, instanceType: Snapshot,
> instanceId: 1, cmd:
> org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotCmd,
> cmdOriginator: null, cmdInfo:
> {"id":"1","response":"json","sessionkey":"YUyQ2NNnnp/GwC0zjw71QxChfHQ\u003d","ctxUserId":"2","httpmethod":"GET","volumeid":"135a3df1-20e1-4b4b-a433-86976ddc9218","_":"1369045371277","ctxAccountId":"2","ctxStartEventId":"70"},
> cmdVersion: 0, callbackType: 0, callbackAddress: null, status: 0,
> processStatus: 0, resultCode: 0, result: null, initMsid: 7332683579487,
> completeMsid: null, lastUpdated: null, lastPolled: null, created: null}
> 2013-05-20 11:51:22,127 DEBUG [cloud.api.ApiServlet] (catalina-exec-13:null)
> ===END=== 10.146.0.15 -- GET
> command=createSnapshot&volumeid=135a3df1-20e1-4b4b-a433-86976ddc9218&response=json&sessionkey=YUyQ2NNnnp%2FGwC0zjw71QxChfHQ%3D&_=1369045371277
> 2013-05-20 11:51:22,131 DEBUG [cloud.async.AsyncJobManagerImpl]
> (Job-Executor-4:job-17) Executing
> org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotCmd for job-17
> 2013-05-20 11:51:22,188 INFO [user.snapshot.CreateSnapshotCmd]
> (Job-Executor-4:job-17) VOLSS: createSnapshotCmd starts:1369065082188
> 2013-05-20 11:51:22,300 DEBUG [agent.transport.Request]
> (Job-Executor-4:job-17) Seq 1-774570943: Sending { Cmd , MgmtId:
> 7332683579487, via: 1, Ver: v1, Flags: 100011,
> [{"org.apache.cloudstack.storage.command.CreateObjectCommand":{"data":{"org.apache.cloudstack.storage.to.SnapshotObjectTO":{"volume":{"uuid":"135a3df1-20e1-4b4b-a433-86976ddc9218","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"c65a038a-750c-3b4f-bf26-7ce3b74e1c85","id":1,"poolType":"NetworkFilesystem","host":"10.147.28.7","path":"/export/home/sanjeev/pri_xen_os","port":2049}},"name":"ROOT-3","size":21474836480,"path":"3fa9a666-20da-4adc-af07-46c92ac1b8f1","volumeId":3,"vmName":"i-2-3-VM","accountId":2,"id":3},"dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"c65a038a-750c-3b4f-bf26-7ce3b74e1c85","id":1,"poolType":"NetworkFilesystem","host":"10.147.28.7","path":"/export/home/sanjeev/pri_xen_os","port":2049}},"vmName":"i-2-3-VM","name":"vm1_ROOT-3_20130520155121","hypervisorType":"XenServer","id":1}},"wait":0}}]
> }
> 2013-05-20 11:51:22,301 DEBUG [agent.transport.Request]
> (Job-Executor-4:job-17) Seq 1-774570943: Executing: { Cmd , MgmtId:
> 7332683579487, via: 1, Ver: v1, Flags: 100011,
> [{"org.apache.cloudstack.storage.command.CreateObjectCommand":{"data":{"org.apache.cloudstack.storage.to.SnapshotObjectTO":{"volume":{"uuid":"135a3df1-20e1-4b4b-a433-86976ddc9218","volumeType":"ROOT","dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"c65a038a-750c-3b4f-bf26-7ce3b74e1c85","id":1,"poolType":"NetworkFilesystem","host":"10.147.28.7","path":"/export/home/sanjeev/pri_xen_os","port":2049}},"name":"ROOT-3","size":21474836480,"path":"3fa9a666-20da-4adc-af07-46c92ac1b8f1","volumeId":3,"vmName":"i-2-3-VM","accountId":2,"id":3},"dataStore":{"org.apache.cloudstack.storage.to.PrimaryDataStoreTO":{"uuid":"c65a038a-750c-3b4f-bf26-7ce3b74e1c85","id":1,"poolType":"NetworkFilesystem","host":"10.147.28.7","path":"/export/home/sanjeev/pri_xen_os","port":2049}},"vmName":"i-2-3-VM","name":"vm1_ROOT-3_20130520155121","hypervisorType":"XenServer","id":1}},"wait":0}}]
> }
> 2013-05-20 11:51:22,301 DEBUG [agent.manager.DirectAgentAttache]
> (DirectAgent-290:null) Seq 1-774570943: Executing request
> 2013-05-20 11:51:24,090 DEBUG [agent.manager.DirectAgentAttache]
> (DirectAgent-290:null) Seq 1-774570943: Response Received:
> 2013-05-20 11:51:24,091 DEBUG [agent.transport.Request]
> (DirectAgent-290:null) Seq 1-774570943: Processing: { Ans: , MgmtId:
> 7332683579487, via: 1, Ver: v1, Flags: 10,
> [{"org.apache.cloudstack.storage.command.CreateObjectAnswer":{"data":{"org.apache.cloudstack.storage.to.SnapshotObjectTO":{"path":"021a0adf-2524-4d15-8cf6-1a09b377fd5c","id":0}},"result":true,"wait":0}}]
> }
> 2013-05-20 11:51:24,091 DEBUG [agent.transport.Request]
> (Job-Executor-4:job-17) Seq 1-774570943: Received: { Ans: , MgmtId:
> 7332683579487, via: 1, Ver: v1, Flags: 10, { CreateObjectAnswer } }
> 2013-05-20 11:51:24,184 DEBUG [storage.snapshot.SnapshotManagerImpl]
> (Job-Executor-4:job-17) Failed to create snapshot
> com.cloud.utils.exception.CloudRuntimeException:
> com.cloud.utils.exception.CloudRuntimeException:
> com.cloud.utils.exception.CloudRuntimeException: Can't find cache storage in
> zone: null
> at
> org.apache.cloudstack.storage.snapshot.SnapshotServiceImpl.backupSnapshot(SnapshotServiceImpl.java:303)
> at
> org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.backupSnapshot(XenserverSnapshotStrategy.java:134)
> at
> org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.takeSnapshot(XenserverSnapshotStrategy.java:208)
> at
> com.cloud.storage.snapshot.SnapshotManagerImpl.takeSnapshot(SnapshotManagerImpl.java:1003)
> at
> org.apache.cloudstack.storage.volume.VolumeServiceImpl.takeSnapshot(VolumeServiceImpl.java:1031)
> at
> com.cloud.storage.VolumeManagerImpl.takeSnapshot(VolumeManagerImpl.java:2601)
> at
> org.apache.cloudstack.api.command.user.snapshot.CreateSnapshotCmd.execute(CreateSnapshotCmd.java:170)
> at com.cloud.api.ApiDispatcher.dispatch(ApiDispatcher.java:155)
> at
> com.cloud.async.AsyncJobManagerImpl$1.run(AsyncJobManagerImpl.java:437)
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
> at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
> at java.util.concurrent.FutureTask.run(FutureTask.java:166)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:679)
> 2013-05-20 11:51:24,201 DEBUG [cloud.async.AsyncJobManagerImpl]
> (Job-Executor-4:job-17) Complete async job-17, jobStatus: 2, resultCode: 530,
> result: Error Code: 530 Error text: Failed to create snapshot due to an
> internal error creating snapshot for volume 3
> 2.Volume state from volumes table:
> mysql> select * from volumes where id=3\G;
> *************************** 1. row ***************************
> id: 3
> account_id: 2
> domain_id: 1
> pool_id: 1
> last_pool_id: NULL
> instance_id: 3
> device_id: 0
> name: ROOT-3
> uuid: 135a3df1-20e1-4b4b-a433-86976ddc9218
> size: 21474836480
> folder: NULL
> path: 3fa9a666-20da-4adc-af07-46c92ac1b8f1
> pod_id: NULL
> data_center_id: 1
> iscsi_name: NULL
> host_ip: NULL
> volume_type: ROOT
> pool_type: NULL
> disk_offering_id: 1
> template_id: 5
> first_snapshot_backup_uuid: NULL
> recreatable: 0
> created: 2013-05-20 10:20:34
> attached: NULL
> updated: 2013-05-20 15:51:24
> removed: NULL
> state: Ready
> chain_info: NULL
> update_count: 4
> disk_type: NULL
> display_volume: 0
> 1 row in set (0.00 sec)
> ERROR:
> No query specified
--
This message is automatically generated by JIRA.
If you think it was sent incorrectly, please contact your JIRA administrators
For more information on JIRA, see: http://www.atlassian.com/software/jira
