[
https://issues.apache.org/jira/browse/CLOUDSTACK-5769?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
angeline shen updated CLOUDSTACK-5769:
--------------------------------------
Attachment: management-server.log.gz
2014-01-06 11:39:42,637 DEBUG [c.c.a.t.Request] (AgentManager-Handler-6:null)
Seq 5-1900284457: Processing: { Ans: , MgmtId: 6958920763321, via: 5, Ver: v1,
Flags: 10, [{"org.apache.cloudstack.sto
rage.command.CopyCmdAnswer":{"result":false,"details":"backup snapshot
exception: Exception: com.cloud.utils.exception.CloudRuntimeException\nMessage:
GetRootDir for nfs://10.223.110.232/export/hom
e/angie/secondary/vmdix64 failed due to
com.cloud.utils.exception.CloudRuntimeException: Unable to mount
10.223.110.232:/export/home/angie/secondary/vmdix64 at
/mnt/SecStorage/38acfefc-ecf2-384a-9f
5f-12b7916e6cd8 due to java.io.IOException: Cannot run program \"mount\":
java.io.IOException: error=12, Cannot allocate memory\n\tat
java.lang.ProcessBuilder.start(ProcessBuilder.java:488)\n\tat c
om.cloud.utils.script.Script.execute(Script.java:177)\n\tat
com.cloud.utils.script.Script.execute(Script.java:155)\n\tat
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.attemptMo
unt(NfsSecondaryStorageResource.java:2385)\n\tat
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.mount(NfsSecondaryStorageResource.java:2342)\n\tat
org.apache.cloudstack.storage.
resource.NfsSecondaryStorageResource.mountUri(NfsSecondaryStorageResource.java:2304)\n\tat
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.getRootDir(NfsSecondaryStorageResource.
java:1945)\n\tat
com.cloud.storage.resource.VmwareSecondaryStorageResourceHandler.getMountPoint(VmwareSecondaryStorageResourceHandler.java:311)\n\tat
com.cloud.storage.resource.VmwareStorageProcess
or.exportVolumeToSecondaryStroage(VmwareStorageProcessor.java:942)\n\tat
com.cloud.storage.resource.VmwareStorageProcessor.backupSnapshotToSecondaryStorage(VmwareStorageProcessor.java:988)\n\tat
co
m.cloud.storage.resource.VmwareStorageProcessor.backupSnapshot(VmwareStorageProcessor.java:1068)\n\tat
com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandH
andlerBase.java:90)\n\tat
com.cloud.storage.resource.VmwareStorageSubsystemCommandHandler.execute(VmwareStorageSubsystemCommandHandler.java:155)\n\tat
com.cloud.storage.resource.StorageSubsystemCom
mandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:50)\n\tat
com.cloud.storage.resource.VmwareSecondaryStorageResourceHandler.executeRequest(VmwareSecondaryStorageResourceHandler.java:101)\n\tat
com.cloud.storage.resource.PremiumSecondaryStorageResource.executeRequest(PremiumSecondaryStorageResource.java:56)\n\tat
com.cloud.agent.Agent.processRequest(Agent.java:498)\n\tat
com.cloud.agent.Agent$AgentRequestHandler.doTask(Agent.java:806)\n\tat
com.cloud.utils.nio.Task.run(Task.java:83)\n\tat
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)\n\tat
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)\n\tat
java.lang.Thread.run(Thread.java:679)\nCaused by: java.io.IOException:
java.io.IOException: error=12, Cannot allocate memory\n\tat
java.lang.UNIXProcess.<init>(UNIXProcess.java:164)\n\tat
java.lang.ProcessImpl.start(ProcessImpl.java:81)\n\tat
java.lang.ProcessBuilder.start(ProcessBuilder.java:470)\n\t... 21
more\n\n","wait":0}}] }
2014-01-06 11:39:42,638 DEBUG [c.c.a.t.Request] (Job-Executor-161:ctx-73fb9eae
ctx-162cc95b) Seq 5-1900284457: Received: { Ans: , MgmtId: 6958920763321, via:
5, Ver: v1, Flags: 10, { CopyCmdAnswer } }
2014-01-06 11:39:42,679 DEBUG [c.c.s.s.SnapshotManagerImpl]
(Job-Executor-162:ctx-f884bd06 ctx-2938ba87) Failed to create snapshot
com.cloud.utils.exception.CloudRuntimeException: backup snapshot exception:
Exception: com.cloud.utils.exception.CloudRuntimeException
Message: GetRootDir for
nfs://10.223.110.232/export/home/angie/secondary/vmdix64 failed due to
com.cloud.utils.exception.CloudRuntimeException: Unable to mount
10.223.110.232:/export/home/angie/secondary/vmdix64 at
/mnt/SecStorage/38acfefc-ecf2-384a-9f5f-12b7916e6cd8 due to
java.io.IOException: Cannot run program "mount": java.io.IOException: error=12,
Cannot allocate memory
at java.lang.ProcessBuilder.start(ProcessBuilder.java:488)
at com.cloud.utils.script.Script.execute(Script.java:177)
at com.cloud.utils.script.Script.execute(Script.java:155)
at
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.attemptMount(NfsSecondaryStorageResource.java:2385)
at
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.mount(NfsSecondaryStorageResource.java:2342)
at
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.mountUri(NfsSecondaryStorageResource.java:2304)
at
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.getRootDir(NfsSecondaryStorageResource.java:1945)
at
com.cloud.storage.resource.VmwareSecondaryStorageResourceHandler.getMountPoint(VmwareSecondaryStorageResourceHandler.java:311)
at
com.cloud.storage.resource.VmwareStorageProcessor.exportVolumeToSecondaryStroage(VmwareStorageProcessor.java:942)
at
com.cloud.storage.resource.VmwareStorageProcessor.backupSnapshotToSecondaryStorage(VmwareStorageProcessor.java:988)
at
com.cloud.storage.resource.VmwareStorageProcessor.backupSnapshot(VmwareStorageProcessor.java:1068)
at
com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandHandlerBase.java:90)
at
com.cloud.storage.resource.VmwareStorageSubsystemCommandHandler.execute(VmwareStorageSubsystemCommandHandler.java:155)
at
com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:50)
at
com.cloud.storage.resource.VmwareSecondaryStorageResourceHandler.executeRequest(VmwareSecondaryStorageResourceHandler.java:101)
at
com.cloud.storage.resource.PremiumSecondaryStorageResource.executeRequest(PremiumSecondaryStorageResource.java:56)
at com.cloud.agent.Agent.processRequest(Agent.java:498)
at com.cloud.agent.Agent$AgentRequestHandler.doTask(Agent.java:806)
at com.cloud.utils.nio.Task.run(Task.java:83)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:679)
Caused by: java.io.IOException: java.io.IOException: error=12, Cannot allocate
memory
at java.lang.UNIXProcess.<init>(UNIXProcess.java:164)
at java.lang.ProcessImpl.start(ProcessImpl.java:81)
at java.lang.ProcessBuilder.start(ProcessBuilder.java:470)
... 21 more
at
org.apache.cloudstack.storage.snapshot.SnapshotServiceImpl.backupSnapshot(SnapshotServiceImpl.java:280)
at
org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.backupSnapshot(XenserverSnapshotStrategy.java:142)
at
org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.takeSnapshot(XenserverSnapshotStrategy.java:301)
at
com.cloud.storage.snapshot.SnapshotManagerImpl.takeSnapshot(SnapshotManagerImpl.java:951)
at sun.reflect.GeneratedMethodAccessor346.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at
org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
at sun.proxy.$Proxy160.takeSnapshot(Unknown Source)
at
org.apache.cloudstack.storage.volume.VolumeServiceImpl.takeSnapshot(VolumeServiceImpl.java:1371)
at
com.cloud.storage.VolumeApiServiceImpl.orchestrateTakeVolumeSnapshot(VolumeApiServiceImpl.java:1725)
at
com.cloud.storage.VolumeApiServiceImpl.orchestrateTakeVolumeSnapshot(VolumeApiServiceImpl.java:2429)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at
com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107)
at
com.cloud.storage.VolumeApiServiceImpl.handleVmWorkJob(VolumeApiServiceImpl.java:2437)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at
org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
at sun.proxy.$Proxy195.handleVmWorkJob(Unknown Source)
at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:99)
at
org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:522)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:679)
2014-01-06 11:39:42,680 DEBUG [c.c.s.s.SnapshotManagerImpl]
(Job-Executor-161:ctx-73fb9eae ctx-162cc95b) Failed to create snapshot
com.cloud.utils.exception.CloudRuntimeException: backup snapshot exception:
Exception: com.cloud.utils.exception.CloudRuntimeException
Message: GetRootDir for
nfs://10.223.110.232/export/home/angie/secondary/vmdix64 failed due to
com.cloud.utils.exception.CloudRuntimeException: Unable to mount
10.223.110.232:/export/home/angie/secondary/vmdix64 at
/mnt/SecStorage/38acfefc-ecf2-384a-9f5f-12b7916e6cd8 due to
java.io.IOException: Cannot run program "mount": java.io.IOException: error=12,
Cannot allocate memory
at java.lang.ProcessBuilder.start(ProcessBuilder.java:488)
at com.cloud.utils.script.Script.execute(Script.java:177)
at com.cloud.utils.script.Script.execute(Script.java:155)
at
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.attemptMount(NfsSecondaryStorageResource.java:2385)
at
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.mount(NfsSecondaryStorageResource.java:2342)
at
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.mountUri(NfsSecondaryStorageResource.java:2304)
at
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.getRootDir(NfsSecondaryStorageResource.java:1945)
at
com.cloud.storage.resource.VmwareSecondaryStorageResourceHandler.getMountPoint(VmwareSecondaryStorageResourceHandler.java:311)
at
com.cloud.storage.resource.VmwareStorageProcessor.exportVolumeToSecondaryStroage(VmwareStorageProcessor.java:942)
at
com.cloud.storage.resource.VmwareStorageProcessor.backupSnapshotToSecondaryStorage(VmwareStorageProcessor.java:988)
at
com.cloud.storage.resource.VmwareStorageProcessor.backupSnapshot(VmwareStorageProcessor.java:1068)
at
com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandHandlerBase.java:90)
at
com.cloud.storage.resource.VmwareStorageSubsystemCommandHandler.execute(VmwareStorageSubsystemCommandHandler.java:155)
at
com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:50)
at
com.cloud.storage.resource.VmwareSecondaryStorageResourceHandler.executeRequest(VmwareSecondaryStorageResourceHandler.java:101)
at
com.cloud.storage.resource.PremiumSecondaryStorageResource.executeRequest(PremiumSecondaryStorageResource.java:56)
at com.cloud.agent.Agent.processRequest(Agent.java:498)
at com.cloud.agent.Agent$AgentRequestHandler.doTask(Agent.java:806)
at com.cloud.utils.nio.Task.run(Task.java:83)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:679)
at java.lang.Thread.run(Thread.java:679)
Caused by: java.io.IOException: java.io.IOException: error=12, Cannot allocate
memory
at java.lang.UNIXProcess.<init>(UNIXProcess.java:164)
at java.lang.ProcessImpl.start(ProcessImpl.java:81)
at java.lang.ProcessBuilder.start(ProcessBuilder.java:470)
... 21 more
at
org.apache.cloudstack.storage.snapshot.SnapshotServiceImpl.backupSnapshot(SnapshotServiceImpl.java:280)
at
org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.backupSnapshot(XenserverSnapshotStrategy.java:142)
at
org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.takeSnapshot(XenserverSnapshotStrategy.java:301)
at
com.cloud.storage.snapshot.SnapshotManagerImpl.takeSnapshot(SnapshotManagerImpl.java:951)
at sun.reflect.GeneratedMethodAccessor346.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at
org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
at sun.proxy.$Proxy160.takeSnapshot(Unknown Source)
at
org.apache.cloudstack.storage.volume.VolumeServiceImpl.takeSnapshot(VolumeServiceImpl.java:1371)
at
com.cloud.storage.VolumeApiServiceImpl.orchestrateTakeVolumeSnapshot(VolumeApiServiceImpl.java:1725)
at
com.cloud.storage.VolumeApiServiceImpl.orchestrateTakeVolumeSnapshot(VolumeApiServiceImpl.java:2429)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at
com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107)
at
com.cloud.storage.VolumeApiServiceImpl.handleVmWorkJob(VolumeApiServiceImpl.java:2437)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at
org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
at sun.proxy.$Proxy195.handleVmWorkJob(Unknown Source)
at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:99)
at
org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:522)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:679)
2014-01-06 11:39:42,698 DEBUG [o.a.c.s.v.VolumeServiceImpl]
(Job-Executor-161:ctx-73fb9eae ctx-162cc95b) Take snapshot: 42 failed
com.cloud.utils.exception.CloudRuntimeException: Failed to create snapshot
at
com.cloud.storage.snapshot.SnapshotManagerImpl.takeSnapshot(SnapshotManagerImpl.java:972)
at sun.reflect.GeneratedMethodAccessor346.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at
org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
at sun.proxy.$Proxy160.takeSnapshot(Unknown Source)
at
org.apache.cloudstack.storage.volume.VolumeServiceImpl.takeSnapshot(VolumeServiceImpl.java:1371)
at
com.cloud.storage.VolumeApiServiceImpl.orchestrateTakeVolumeSnapshot(VolumeApiServiceImpl.java:1725)
at
com.cloud.storage.VolumeApiServiceImpl.orchestrateTakeVolumeSnapshot(VolumeApiServiceImpl.java:2429)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at
com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107)
at
com.cloud.storage.VolumeApiServiceImpl.handleVmWorkJob(VolumeApiServiceImpl.java:2437)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at
org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
at sun.proxy.$Proxy195.handleVmWorkJob(Unknown Source)
at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:99)
at
org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:522)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:679)
Caused by: com.cloud.utils.exception.CloudRuntimeException: backup snapshot
exception: Exception: com.cloud.utils.exception.CloudRuntimeException
Message: GetRootDir for
nfs://10.223.110.232/export/home/angie/secondary/vmdix64 failed due to
com.cloud.utils.exception.CloudRuntimeException: Unable to mount
10.223.110.232:/export/home/angie/secondary/vmdix64 at
/mnt/SecStorage/38acfefc-ecf2-384a-9f5f-12b7916e6cd8 due to
java.io.IOException: Cannot run program "mount": java.io.IOException: error=12,
Cannot allocate memory
at java.lang.ProcessBuilder.start(ProcessBuilder.java:488)
at com.cloud.utils.script.Script.execute(Script.java:177)
at com.cloud.utils.script.Script.execute(Script.java:155)
at
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.attemptMount(NfsSecondaryStorageResource.java:2385)
at
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.mount(NfsSecondaryStorageResource.java:2342)
at
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.mountUri(NfsSecondaryStorageResource.java:2304)
at
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.getRootDir(NfsSecondaryStorageResource.java:1945)
at
com.cloud.storage.resource.VmwareSecondaryStorageResourceHandler.getMountPoint(VmwareSecondaryStorageResourceHandler.java:311)
at
com.cloud.storage.resource.VmwareStorageProcessor.exportVolumeToSecondaryStroage(VmwareStorageProcessor.java:942)
at
com.cloud.storage.resource.VmwareStorageProcessor.backupSnapshotToSecondaryStorage(VmwareStorageProcessor.java:988)
at
com.cloud.storage.resource.VmwareStorageProcessor.backupSnapshot(VmwareStorageProcessor.java:1068)
at
com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandHandlerBase.java:90)
at
com.cloud.storage.resource.VmwareStorageSubsystemCommandHandler.execute(VmwareStorageSubsystemCommandHandler.java:155)
at
com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:50)
at
com.cloud.storage.resource.VmwareSecondaryStorageResourceHandler.executeRequest(VmwareSecondaryStorageResourceHandler.java:101)
at
com.cloud.storage.resource.PremiumSecondaryStorageResource.executeRequest(PremiumSecondaryStorageResource.java:56)
at com.cloud.agent.Agent.processRequest(Agent.java:498)
at com.cloud.agent.Agent$AgentRequestHandler.doTask(Agent.java:806)
at com.cloud.utils.nio.Task.run(Task.java:83)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:679)
Caused by: java.io.IOException: java.io.IOException: error=12, Cannot allocate
memory
at java.lang.UNIXProcess.<init>(UNIXProcess.java:164)
:
at java.lang.UNIXProcess.<init>(UNIXProcess.java:164)
at java.lang.ProcessImpl.start(ProcessImpl.java:81)
at java.lang.ProcessBuilder.start(ProcessBuilder.java:470)
... 21 more
at
org.apache.cloudstack.storage.snapshot.SnapshotServiceImpl.backupSnapshot(SnapshotServiceImpl.java:280)
at
org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.backupSnapshot(XenserverSnapshotStrategy.java:142)
at
org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.takeSnapshot(XenserverSnapshotStrategy.java:301)
at
com.cloud.storage.snapshot.SnapshotManagerImpl.takeSnapshot(SnapshotManagerImpl.java:951)
... 43 more
2014-01-06 11:39:42,700 DEBUG [o.a.c.s.v.VolumeServiceImpl]
(Job-Executor-162:ctx-f884bd06 ctx-2938ba87) Take snapshot: 28 failed
com.cloud.utils.exception.CloudRuntimeException: Failed to create snapshot
at
com.cloud.storage.snapshot.SnapshotManagerImpl.takeSnapshot(SnapshotManagerImpl.java:972)
at sun.reflect.GeneratedMethodAccessor346.invoke(Unknown Source)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at
org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
at sun.proxy.$Proxy160.takeSnapshot(Unknown Source)
at
org.apache.cloudstack.storage.volume.VolumeServiceImpl.takeSnapshot(VolumeServiceImpl.java:1371)
at
com.cloud.storage.VolumeApiServiceImpl.orchestrateTakeVolumeSnapshot(VolumeApiServiceImpl.java:1725)
at
com.cloud.storage.VolumeApiServiceImpl.orchestrateTakeVolumeSnapshot(VolumeApiServiceImpl.java:2429)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at
com.cloud.vm.VmWorkJobHandlerProxy.handleVmWorkJob(VmWorkJobHandlerProxy.java:107)
at
com.cloud.storage.VolumeApiServiceImpl.handleVmWorkJob(VolumeApiServiceImpl.java:2437)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:616)
at
org.springframework.aop.support.AopUtils.invokeJoinpointUsingReflection(AopUtils.java:317)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.invokeJoinpoint(ReflectiveMethodInvocation.java:183)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:150)
at
org.springframework.aop.interceptor.ExposeInvocationInterceptor.invoke(ExposeInvocationInterceptor.java:91)
at
org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
at
org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:204)
at sun.proxy.$Proxy195.handleVmWorkJob(Unknown Source)
at com.cloud.vm.VmWorkJobDispatcher.runJob(VmWorkJobDispatcher.java:99)
at
org.apache.cloudstack.framework.jobs.impl.AsyncJobManagerImpl$5.runInContext(AsyncJobManagerImpl.java:522)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable$1.run(ManagedContextRunnable.java:49)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext$1.call(DefaultManagedContext.java:56)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.callWithContext(DefaultManagedContext.java:103)
at
org.apache.cloudstack.managed.context.impl.DefaultManagedContext.runWithContext(DefaultManagedContext.java:53)
at
org.apache.cloudstack.managed.context.ManagedContextRunnable.run(ManagedContextRunnable.java:46)
at
java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:471)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:334)
at java.util.concurrent.FutureTask.run(FutureTask.java:166)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:679)
Caused by: com.cloud.utils.exception.CloudRuntimeException: backup snapshot
exception: Exception: com.cloud.utils.exception.CloudRuntimeException
Message: GetRootDir for
nfs://10.223.110.232/export/home/angie/secondary/vmdix64 failed due to
com.cloud.utils.exception.CloudRuntimeException: Unable to mount
10.223.110.232:/export/home/angie/secondary/vmdix64 at
/mnt/SecStorage/38acfefc-ecf2-384a-9f5f-12b7916e6cd8 due to
java.io.IOException: Cannot run program "mount": java.io.IOException: error=12,
Cannot allocate memory
at java.lang.ProcessBuilder.start(ProcessBuilder.java:488)
at com.cloud.utils.script.Script.execute(Script.java:177)
at com.cloud.utils.script.Script.execute(Script.java:155)
at
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.attemptMount(NfsSecondaryStorageResource.java:2385)
at
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.mount(NfsSecondaryStorageResource.java:2342)
at
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.mount(NfsSecondaryStorageResource.java:2342)
at
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.mountUri(NfsSecondaryStorageResource.java:2304)
at
org.apache.cloudstack.storage.resource.NfsSecondaryStorageResource.getRootDir(NfsSecondaryStorageResource.java:1945)
at
com.cloud.storage.resource.VmwareSecondaryStorageResourceHandler.getMountPoint(VmwareSecondaryStorageResourceHandler.java:311)
at
com.cloud.storage.resource.VmwareStorageProcessor.exportVolumeToSecondaryStroage(VmwareStorageProcessor.java:942)
at
com.cloud.storage.resource.VmwareStorageProcessor.backupSnapshotToSecondaryStorage(VmwareStorageProcessor.java:988)
at
com.cloud.storage.resource.VmwareStorageProcessor.backupSnapshot(VmwareStorageProcessor.java:1068)
at
com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.execute(StorageSubsystemCommandHandlerBase.java:90)
at
com.cloud.storage.resource.VmwareStorageSubsystemCommandHandler.execute(VmwareStorageSubsystemCommandHandler.java:155)
at
com.cloud.storage.resource.StorageSubsystemCommandHandlerBase.handleStorageCommands(StorageSubsystemCommandHandlerBase.java:50)
at
com.cloud.storage.resource.VmwareSecondaryStorageResourceHandler.executeRequest(VmwareSecondaryStorageResourceHandler.java:101)
at
com.cloud.storage.resource.PremiumSecondaryStorageResource.executeRequest(PremiumSecondaryStorageResource.java:56)
at com.cloud.agent.Agent.processRequest(Agent.java:498)
at com.cloud.agent.Agent$AgentRequestHandler.doTask(Agent.java:806)
at com.cloud.utils.nio.Task.run(Task.java:83)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
at java.lang.Thread.run(Thread.java:679)
Caused by: java.io.IOException: java.io.IOException: error=12, Cannot allocate
memory
at java.lang.UNIXProcess.<init>(UNIXProcess.java:164)
at java.lang.ProcessImpl.start(ProcessImpl.java:81)
at java.lang.ProcessBuilder.start(ProcessBuilder.java:470)
... 21 more
at
org.apache.cloudstack.storage.snapshot.SnapshotServiceImpl.backupSnapshot(SnapshotServiceImpl.java:280)
at
org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.backupSnapshot(XenserverSnapshotStrategy.java:142)
at
org.apache.cloudstack.storage.snapshot.XenserverSnapshotStrategy.takeSnapshot(XenserverSnapshotStrategy.java:301)
at
com.cloud.storage.snapshot.SnapshotManagerImpl.takeSnapshot(SnapshotManagerImpl.java:951)
... 43 more
2014-01-06 11:39:42,701 DEBUG [c.c.v.VmWorkJobHandlerProxy]
(Job-Executor-162:ctx-f884bd06 ctx-2938ba87) Done executing VM work job:
com.cloud.storage.VmWorkTakeVolumeSnapshot{"volumeId":28,"policyId":8,"snapshotId":236,"quiesceVm":false,"userId":1,"accountId":2,"vmId":15,"handlerName":"VolumeApiServiceImpl"}
2014-01-06 11:39:42,701 DEBUG [o.a.c.f.j.i.AsyncJobManagerImpl]
(Job-Executor-162:ctx-f884bd06 ctx-2938ba87) Complete async job-696, jobStatus:
SUCCEEDED, resultCode: 0, result:
rO0ABXNyAA5qYXZhLmxhbmcuTG9uZzuL5JDMjyPfAgABSgAFdmFsdWV4cgAQamF2YS5sYW5nLk51bWJlcoaslR0LlOCLAgAAeHAAAAAAAAAA7A
> [vmware] Vcenter 5.5 host ESXi 5.5 scheduled snapshot failed
> -------------------------------------------------------------
>
> Key: CLOUDSTACK-5769
> URL: https://issues.apache.org/jira/browse/CLOUDSTACK-5769
> Project: CloudStack
> Issue Type: Bug
> Security Level: Public(Anyone can view this level - this is the
> default.)
> Components: Management Server
> Affects Versions: 4.3.0
> Environment: MS build 4.3 1/2/2014
> vcenter 5.5 host ESXi 5.5
> Reporter: angeline shen
> Assignee: Likitha Shetty
> Priority: Critical
> Fix For: 4.3.0
>
> Attachments: management-server.log.gz, management-server.log.gz
>
>
> 1. advance zone, create VMs
> 2. create scheduled snapshot.
> Test result: snapshots generated remain in error state
--
This message was sent by Atlassian JIRA
(v6.1.5#6160)
