# root@s-340-VM:~# cat /var/cache/cloud/cmdline
template=domP type=secstorage host=10.101.2.40 port=8250 name=s-340-VM zone=1
pod=1 guid=s-340-VM workers=5 authorized_key=****
resource=com.cloud.storage.resource.PremiumSecondaryStorageResource
instance=SecStorage sslcopy=true role=templateProcessor mtu=1500
eth2ip=148.59.36.60 eth2mask=255.255.255.240 gateway=148.59.36.49
public.network.device=eth2 eth0ip=169.254.211.29 eth0mask=255.255.0.0
eth1ip=10.101.3.231 eth1mask=255.255.252.0 mgmtcidr=10.101.0.0/22
localgw=10.101.0.1 private.network.device=eth1 eth3ip=10.101.7.212
eth3mask=255.255.254.0 storageip=10.101.7.212 storagenetmask=255.255.254.0
storagegateway=10.101.6.1 internaldns1=10.101.0.1 dns1=1.1.1.1 dns2=8.8.8.8
nfsVersion=null keystore_password=*****
# cat /var/log/cloudstack/management/management-server.log.2023-02-*.gz | zgrep
SecStorageSetupCommand
2023-02-18 14:35:38,699 DEBUG [c.c.a.t.Request]
(AgentConnectTaskPool-290:ctx-cf94f90e) (logid:6dc1b961) Seq
47-6546545008336437249: Sending { Cmd , MgmtId: 130593671224, via:
47(s-292-VM), Ver: v1, Flags: 100111,
[{"com.cloud.agent.api.SecStorageSetupCommand":{"store":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.91.6.5/volume1/ACS_Backup06","_role":"Image"}},"secUrl":"nfs://10.91.6.5/volume1/ACS_Backup06","certs":{},"postUploadKey":"89HVaWPMPwbWI-QGJrI5jzoGULt3lyZzHN4pnc-kn36Le5Hy_Hh3l6ZABLMgJXeBlA4vspDa-NyrxtBbJGj20A","wait":"0","bypassHostMaintenance":"false"}}]
}
2023-02-18 14:35:42,024 DEBUG [c.c.a.m.AgentManagerImpl]
(AgentConnectTaskPool-290:ctx-cf94f90e) (logid:6dc1b961) Details from executing
class com.cloud.agent.api.SecStorageSetupCommand: success
2023-02-20 11:12:33,345 DEBUG [c.c.a.t.Request]
(AgentConnectTaskPool-4:ctx-3b91dfcf) (logid:50ea75a6) Seq
43-719450040472436737: Sending { Cmd , MgmtId: 130593671224, via:
43(s-289-VM), Ver: v1, Flags: 100111,
[{"com.cloud.agent.api.SecStorageSetupCommand":{"store":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.101.6.40/export/secondary","_role":"Image"}},"secUrl":"nfs://10.101.6.40/export/secondary","certs":{},"postUploadKey":"89HVaWPMPwbWI-QGJrI5jzoGULt3lyZzHN4pnc-kn36Le5Hy_Hh3l6ZABLMgJXeBlA4vspDa-NyrxtBbJGj20A","wait":"0","bypassHostMaintenance":"false"}}]
}
2023-02-20 11:12:34,389 DEBUG [c.c.a.t.Request]
(AgentConnectTaskPool-5:ctx-5834440f) (logid:2d16c04c) Seq
47-4507540277044445185: Sending { Cmd , MgmtId: 130593671224, via:
47(s-292-VM), Ver: v1, Flags: 100111,
[{"com.cloud.agent.api.SecStorageSetupCommand":{"store":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.91.6.5/volume1/ACS_Backup06","_role":"Image"}},"secUrl":"nfs://10.91.6.5/volume1/ACS_Backup06","certs":{},"postUploadKey":"89HVaWPMPwbWI-QGJrI5jzoGULt3lyZzHN4pnc-kn36Le5Hy_Hh3l6ZABLMgJXeBlA4vspDa-NyrxtBbJGj20A","wait":"0","bypassHostMaintenance":"false"}}]
}
2023-02-20 11:12:37,406 DEBUG [c.c.a.m.AgentManagerImpl]
(AgentConnectTaskPool-5:ctx-5834440f) (logid:2d16c04c) Details from executing
class com.cloud.agent.api.SecStorageSetupCommand: success
2023-02-20 11:59:14,400 WARN [c.c.a.m.AgentAttache]
(AgentConnectTaskPool-4:ctx-3b91dfcf) (logid:50ea75a6) Seq
43-719450040472436737: Timed out on Seq 43-719450040472436737: { Cmd , MgmtId:
130593671224, via: 43(s-289-VM), Ver: v1, Flags: 10100111,
[{"com.cloud.agent.api.SecStorageSetupCommand":{"store":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.101.6.40/export/secondary","_role":"Image"}},"secUrl":"nfs://10.101.6.40/export/secondary","certs":{},"postUploadKey":"89HVaWPMPwbWI-QGJrI5jzoGULt3lyZzHN4pnc-kn36Le5Hy_Hh3l6ZABLMgJXeBlA4vspDa-NyrxtBbJGj20A","wait":"0","bypassHostMaintenance":"false"}}]
}
2023-02-20 12:25:40,060 DEBUG [c.c.a.t.Request]
(AgentConnectTaskPool-16:ctx-236c5a52) (logid:e701c43f) Seq
48-8498011021871415297: Sending { Cmd , MgmtId: 130593671224, via:
48(s-311-VM), Ver: v1, Flags: 100111,
[{"com.cloud.agent.api.SecStorageSetupCommand":{"store":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.101.6.40/export/secondary","_role":"Image"}},"secUrl":"nfs://10.101.6.40/export/secondary","certs":{},"postUploadKey":"89HVaWPMPwbWI-QGJrI5jzoGULt3lyZzHN4pnc-kn36Le5Hy_Hh3l6ZABLMgJXeBlA4vspDa-NyrxtBbJGj20A","wait":"0","bypassHostMaintenance":"false"}}]
}
2023-02-20 12:25:43,138 DEBUG [c.c.a.m.AgentManagerImpl]
(AgentConnectTaskPool-16:ctx-236c5a52) (logid:e701c43f) Details from executing
class com.cloud.agent.api.SecStorageSetupCommand: success
2023-02-20 12:25:43,159 DEBUG [c.c.a.t.Request]
(AgentConnectTaskPool-16:ctx-236c5a52) (logid:e701c43f) Seq
48-8498011021871415298: Sending { Cmd , MgmtId: 130593671224, via:
48(s-311-VM), Ver: v1, Flags: 100111,
[{"com.cloud.agent.api.SecStorageSetupCommand":{"store":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.101.6.23/mnt/Store08/CSBackup08","_role":"Image"}},"secUrl":"nfs://10.101.6.23/mnt/Store08/CSBackup08","certs":{},"postUploadKey":"89HVaWPMPwbWI-QGJrI5jzoGULt3lyZzHN4pnc-kn36Le5Hy_Hh3l6ZABLMgJXeBlA4vspDa-NyrxtBbJGj20A","wait":"0","bypassHostMaintenance":"false"}}]
}
2023-02-20 12:25:45,730 DEBUG [c.c.a.m.AgentManagerImpl]
(AgentConnectTaskPool-16:ctx-236c5a52) (logid:e701c43f) Details from executing
class com.cloud.agent.api.SecStorageSetupCommand: success
2023-02-20 12:53:59,308 DEBUG [c.c.a.t.Request]
(AgentConnectTaskPool-10:ctx-bf90573f) (logid:dc87251d) Seq
48-3231051257661620225: Sending { Cmd , MgmtId: 130593671224, via:
48(s-311-VM), Ver: v1, Flags: 100111,
[{"com.cloud.agent.api.SecStorageSetupCommand":{"store":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.101.6.40/export/secondary","_role":"Image"}},"secUrl":"nfs://10.101.6.40/export/secondary","certs":{},"postUploadKey":"89HVaWPMPwbWI-QGJrI5jzoGULt3lyZzHN4pnc-kn36Le5Hy_Hh3l6ZABLMgJXeBlA4vspDa-NyrxtBbJGj20A","wait":"0","bypassHostMaintenance":"false"}}]
}
2023-02-20 12:54:01,842 DEBUG [c.c.a.m.AgentManagerImpl]
(AgentConnectTaskPool-10:ctx-bf90573f) (logid:dc87251d) Details from executing
class com.cloud.agent.api.SecStorageSetupCommand: success
2023-02-20 12:54:01,871 DEBUG [c.c.a.t.Request]
(AgentConnectTaskPool-10:ctx-bf90573f) (logid:dc87251d) Seq
48-3231051257661620226: Sending { Cmd , MgmtId: 130593671224, via:
48(s-311-VM), Ver: v1, Flags: 100111,
[{"com.cloud.agent.api.SecStorageSetupCommand":{"store":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.101.6.23/mnt/Store08/CSBackup08","_role":"Image"}},"secUrl":"nfs://10.101.6.23/mnt/Store08/CSBackup08","certs":{},"postUploadKey":"89HVaWPMPwbWI-QGJrI5jzoGULt3lyZzHN4pnc-kn36Le5Hy_Hh3l6ZABLMgJXeBlA4vspDa-NyrxtBbJGj20A","wait":"0","bypassHostMaintenance":"false"}}]
}
2023-02-20 12:54:04,295 DEBUG [c.c.a.m.AgentManagerImpl]
(AgentConnectTaskPool-10:ctx-bf90573f) (logid:dc87251d) Details from executing
class com.cloud.agent.api.SecStorageSetupCommand: success
2023-02-22 15:23:33,561 DEBUG [c.c.a.t.Request]
(AgentConnectTaskPool-23:ctx-22a41bf8) (logid:894a50f0) Seq
50-2542563464627355649: Sending { Cmd , MgmtId: 130593671224, via:
50(s-324-VM), Ver: v1, Flags: 100111,
[{"com.cloud.agent.api.SecStorageSetupCommand":{"store":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.91.6.5/volume1/ACS_Backup06","_role":"Image"}},"secUrl":"nfs://10.91.6.5/volume1/ACS_Backup06","certs":{},"postUploadKey":"89HVaWPMPwbWI-QGJrI5jzoGULt3lyZzHN4pnc-kn36Le5Hy_Hh3l6ZABLMgJXeBlA4vspDa-NyrxtBbJGj20A","wait":"0","bypassHostMaintenance":"false"}}]
}
2023-02-22 15:23:36,815 DEBUG [c.c.a.m.AgentManagerImpl]
(AgentConnectTaskPool-23:ctx-22a41bf8) (logid:894a50f0) Details from executing
class com.cloud.agent.api.SecStorageSetupCommand: success
2023-02-26 14:46:39,737 DEBUG [c.c.a.t.Request]
(AgentConnectTaskPool-26:ctx-50d91205) (logid:4c7783a0) Seq
52-8409064929230848001: Sending { Cmd , MgmtId: 130593671224, via:
52(s-339-VM), Ver: v1, Flags: 100111,
[{"com.cloud.agent.api.SecStorageSetupCommand":{"store":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.101.6.40/export/secondary","_role":"Image"}},"secUrl":"nfs://10.101.6.40/export/secondary","certs":{},"postUploadKey":"89HVaWPMPwbWI-QGJrI5jzoGULt3lyZzHN4pnc-kn36Le5Hy_Hh3l6ZABLMgJXeBlA4vspDa-NyrxtBbJGj20A","wait":"0","bypassHostMaintenance":"false"}}]
}
2023-02-26 14:46:42,926 DEBUG [c.c.a.m.AgentManagerImpl]
(AgentConnectTaskPool-26:ctx-50d91205) (logid:4c7783a0) Details from executing
class com.cloud.agent.api.SecStorageSetupCommand: success
2023-02-26 14:46:42,945 DEBUG [c.c.a.t.Request]
(AgentConnectTaskPool-26:ctx-50d91205) (logid:4c7783a0) Seq
52-8409064929230848002: Sending { Cmd , MgmtId: 130593671224, via:
52(s-339-VM), Ver: v1, Flags: 100111,
[{"com.cloud.agent.api.SecStorageSetupCommand":{"store":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.101.6.23/mnt/Store08/CSBackup08","_role":"Image"}},"secUrl":"nfs://10.101.6.23/mnt/Store08/CSBackup08","certs":{},"postUploadKey":"89HVaWPMPwbWI-QGJrI5jzoGULt3lyZzHN4pnc-kn36Le5Hy_Hh3l6ZABLMgJXeBlA4vspDa-NyrxtBbJGj20A","wait":"0","bypassHostMaintenance":"false"}}]
}
2023-02-26 14:46:45,435 DEBUG [c.c.a.m.AgentManagerImpl]
(AgentConnectTaskPool-26:ctx-50d91205) (logid:4c7783a0) Details from executing
class com.cloud.agent.api.SecStorageSetupCommand: success
2023-02-26 15:07:11,934 DEBUG [c.c.a.t.Request]
(AgentConnectTaskPool-27:ctx-8f2e92c1) (logid:4a947f5e) Seq
52-7356067041356283905: Sending { Cmd , MgmtId: 130593671224, via:
52(s-339-VM), Ver: v1, Flags: 100111,
[{"com.cloud.agent.api.SecStorageSetupCommand":{"store":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.101.6.40/export/secondary","_role":"Image"}},"secUrl":"nfs://10.101.6.40/export/secondary","certs":{},"postUploadKey":"89HVaWPMPwbWI-QGJrI5jzoGULt3lyZzHN4pnc-kn36Le5Hy_Hh3l6ZABLMgJXeBlA4vspDa-NyrxtBbJGj20A","wait":"0","bypassHostMaintenance":"false"}}]
}
2023-02-26 15:07:14,985 DEBUG [c.c.a.m.AgentManagerImpl]
(AgentConnectTaskPool-27:ctx-8f2e92c1) (logid:4a947f5e) Details from executing
class com.cloud.agent.api.SecStorageSetupCommand: success
2023-02-26 15:07:15,001 DEBUG [c.c.a.t.Request]
(AgentConnectTaskPool-27:ctx-8f2e92c1) (logid:4a947f5e) Seq
52-7356067041356283906: Sending { Cmd , MgmtId: 130593671224, via:
52(s-339-VM), Ver: v1, Flags: 100111,
[{"com.cloud.agent.api.SecStorageSetupCommand":{"store":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.101.6.23/mnt/Store08/CSBackup08","_role":"Image"}},"secUrl":"nfs://10.101.6.23/mnt/Store08/CSBackup08","certs":{},"postUploadKey":"89HVaWPMPwbWI-QGJrI5jzoGULt3lyZzHN4pnc-kn36Le5Hy_Hh3l6ZABLMgJXeBlA4vspDa-NyrxtBbJGj20A","wait":"0","bypassHostMaintenance":"false"}}]
}
2023-02-26 15:07:17,516 DEBUG [c.c.a.m.AgentManagerImpl]
(AgentConnectTaskPool-27:ctx-8f2e92c1) (logid:4a947f5e) Details from executing
class com.cloud.agent.api.SecStorageSetupCommand: success
2023-02-26 16:03:33,807 DEBUG [c.c.a.t.Request]
(AgentConnectTaskPool-28:ctx-8b4b3cb8) (logid:6ae260b3) Seq
53-603482350067646465: Sending { Cmd , MgmtId: 130593671224, via:
53(s-340-VM), Ver: v1, Flags: 100111,
[{"com.cloud.agent.api.SecStorageSetupCommand":{"store":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.101.6.40/export/secondary","_role":"Image"}},"secUrl":"nfs://10.101.6.40/export/secondary","certs":{},"postUploadKey":"89HVaWPMPwbWI-QGJrI5jzoGULt3lyZzHN4pnc-kn36Le5Hy_Hh3l6ZABLMgJXeBlA4vspDa-NyrxtBbJGj20A","wait":"0","bypassHostMaintenance":"false"}}]
}
2023-02-26 16:03:37,126 DEBUG [c.c.a.m.AgentManagerImpl]
(AgentConnectTaskPool-28:ctx-8b4b3cb8) (logid:6ae260b3) Details from executing
class com.cloud.agent.api.SecStorageSetupCommand: success
2023-02-26 16:03:37,142 DEBUG [c.c.a.t.Request]
(AgentConnectTaskPool-28:ctx-8b4b3cb8) (logid:6ae260b3) Seq
53-603482350067646466: Sending { Cmd , MgmtId: 130593671224, via:
53(s-340-VM), Ver: v1, Flags: 100111,
[{"com.cloud.agent.api.SecStorageSetupCommand":{"store":{"com.cloud.agent.api.to.NfsTO":{"_url":"nfs://10.101.6.23/mnt/Store08/CSBackup08","_role":"Image"}},"secUrl":"nfs://10.101.6.23/mnt/Store08/CSBackup08","certs":{},"postUploadKey":"89HVaWPMPwbWI-QGJrI5jzoGULt3lyZzHN4pnc-kn36Le5Hy_Hh3l6ZABLMgJXeBlA4vspDa-NyrxtBbJGj20A","wait":"0","bypassHostMaintenance":"false"}}]
}
2023-02-26 16:03:39,890 DEBUG [c.c.a.m.AgentManagerImpl]
(AgentConnectTaskPool-28:ctx-8b4b3cb8) (logid:6ae260b3) Details from executing
class com.cloud.agent.api.SecStorageSetupCommand: success
Antoine Boucher
> On Feb 28, 2023, at 4:47 AM, Wei ZHOU <ustcweiz...@gmail.com> wrote:
>
> The routes should use eth3 not eth1.
>
> Can you share the `/var/cache/cloud/cmdline` file in SSVM, and filter
> management-server.log by keyword `SecStorageSetupCommand` ?
>
>
> -Wei
>
> On Tue, 28 Feb 2023 at 10:42, Granwille Strauss
> <granwi...@namhost.com.invalid <mailto:granwi...@namhost.com.invalid>> wrote:
>
>> I recently had a similar issue and now that I look at my routing tables,
>> storage goes via eth1 and not eth3. Full details on this here:
>> https://github.com/apache/cloudstack/issues/7244#issuecomment-1434755523
>> This, therefore, also explains why I randomly got this error with my SSVM.
>> On 2/28/23 11:35, Daan Hoogland wrote:
>>
>> does sound like a bug Antoine,
>> I did the network calculation and it seems you are right.
>> I wonder about the last two routes as well. did you do anything for those?
>>
>> On Sun, Feb 26, 2023 at 9:47 PM Antoine Boucher <antoi...@haltondc.com
>> <mailto:antoi...@haltondc.com>> <antoi...@haltondc.com
>> <mailto:antoi...@haltondc.com>>
>> wrote:
>>
>>
>> Hello,
>>
>> I'm having a networking issue on SSVMs, I have the following networks
>> defined in “Zone 1”.
>>
>> Management: 10.101.0.0/22
>> Storage: 10.101.6.0/23
>>
>> All worked well until we decided to configure new storage devices on
>> 10.101.7.x, the hosts and management server have no issue but the SSVM is
>> not able to reach it. Here are the defined interfaces of the SSVM and the
>> routing table of the SSVM:
>>
>> 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group
>> default qlen 1000
>> link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
>> inet 127.0.0.1/8 scope host lo
>> valid_lft forever preferred_lft forever
>> 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state
>> UP group default qlen 1000
>> link/ether 0e:00:a9:fe:72:ec brd ff:ff:ff:ff:ff:ff
>> altname enp0s3
>> altname ens3
>> inet 169.254.114.236/16 brd 169.254.255.255 scope global eth0
>> valid_lft forever preferred_lft forever
>> 3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state
>> UP group default qlen 1000
>> link/ether 1e:00:a1:00:00:06 brd ff:ff:ff:ff:ff:ff
>> altname enp0s4
>> altname ens4
>> inet 10.101.3.205/22 brd 10.101.3.255 scope global eth1
>> valid_lft forever preferred_lft forever
>> 4: eth2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state
>> UP group default qlen 1000
>> link/ether 1e:00:6b:00:00:3d brd ff:ff:ff:ff:ff:ff
>> altname enp0s5
>> altname ens5
>> inet 148.59.36.61/28 brd 148.59.36.63 scope global eth2
>> valid_lft forever preferred_lft forever
>> 5: eth3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state
>> UP group default qlen 1000
>> link/ether 1e:00:09:00:00:6b brd ff:ff:ff:ff:ff:ff
>> altname enp0s6
>> altname ens6
>> inet 10.101.7.226/23 brd 10.101.7.255 scope global eth3
>> valid_lft forever preferred_lft forever
>>
>> default via 148.59.36.49 dev eth210.0.0.0/8 via 10.101.0.1 dev
>> eth110.91.0.0/23 via 10.101.0.1 dev eth110.91.6.0/24 via 10.101.0.1 dev
>> eth110.101.0.0/22 dev eth1 proto kernel scope link src
>> 10.101.3.25310.101.6.0/23 via 10.101.0.1 dev eth1148.59.36.48/28 dev eth2
>> proto kernel scope link src 148.59.36.61169.254.0.0/16 dev eth0 proto kernel
>> scope link src 169.254.232.208172.16.0.0/12 via 10.101.0.1 dev
>> eth1192.168.0.0/16 via 10.101.0.1 dev eth1
>>
>> Why is the routing for 10.101.6.0/23 routing via eth1, shoudn’nt it be
>> using eth3? The router seems to be bypassing the routing rules for
>> 10.101.6.x since I see no traffic going through the gateway but I see
>> traffic going through the gateway when the destination is 10.101.7.x
>>
>> If I modify the routing for 10.101.6.0/23 to eth3 all is well.
>>
>> Is this by design?
>>
>> Regards,
>> Antoine Boucher
>>
>>
>> --
>> Regards / Groete
>>
>> <https://www.namhost.com <https://www.namhost.com/>> Granwille Strauss //
>> Senior Systems Admin
>>
>> *e:* granwi...@namhost.com <mailto:granwi...@namhost.com>
>> *m:* +264 81 323 1260 <+264813231260>
>> *w:* www.namhost.com <http://www.namhost.com/>
>>
>> <https://www.facebook.com/namhost> <https://twitter.com/namhost>
>> <https://www.instagram.com/namhostinternetservices/>
>> <https://www.linkedin.com/company/namhos>
>> <https://www.youtube.com/channel/UCTd5v-kVPaic_dguGur15AA>
>>
>>
>> <https://www.adsigner.com/v1/l/631091998d4670001fe43ec2/621c9b76c140bb001ed0f818/banner>
>>
>> Namhost Internet Services (Pty) Ltd,
>>
>> 24 Black Eagle Rd, Hermanus, 7210, RSA
>>
>>
>>
>> The content of this message is confidential. If you have received it by
>> mistake, please inform us by email reply and then delete the message. It is
>> forbidden to copy, forward, or in any way reveal the contents of this
>> message to anyone without our explicit consent. The integrity and security
>> of this email cannot be guaranteed over the Internet. Therefore, the sender
>> will not be held liable for any damage caused by the message. For our full
>> privacy policy and disclaimers, please go to
>> https://www.namhost.com/privacy-policy
>>
>> [image: Powered by AdSigner]
>> <https://www.adsigner.com/v1/c/631091998d4670001fe43ec2/621c9b76c140bb001ed0f818>
