[ovirt-users] Help! My hosted engine lost his nic!
Sorry for the repost, I don't know if the previous message got to the
list...
Hi, I upgraded an oVirt installation a month ago to the latest 3.6.7.
Before it was 3.6.0 if I remember correctly.
Everything went fine so far for a month or so.
A couple of days ago the the default gateway got rebooted and the
physical server hosting the HE decided to shut down the vm because it
could not ping the gateway.
The other host restarted the hevm but it now has *no nic*.
As a workaround I attached a virtio nic via virsh but every time the vm
gets restarted the nic get lost
After a bit of troubleshooting and digging this is what I found:
This is the /var/run/ovirt-hosted-engine-ha/vm.conf which, as far as I
understand, gets extracted from the HE storage domain
emulatedMachine=pc
vmId=497f5e4a-0c76-441a-b72e-724d7092d07e
smp=2
memSize=6144
spiceSecureChannels=smain,sdisplay,sinputs,scursor,splayback,srecord,ssmartcard,susbredir
vmName=HostedEngine
display=vnc
devices={index:0,iface:virtio,format:raw,bootOrder:1,address:{slot:0x06,bus:0x00,domain:0x,type:pci,function:0x0},volumeID:bb3218ba-cbe9-4cd0-b50b-931deae992f7,imageID:d65b82e2-2ad1-
4f4f-bfad-0277c37f2808,readonly:false,domainID:2c3585cc-b7bc-4881-85b3-aa6514991a26,deviceId:d65b82e2-2ad1-4f4f-bfad-0277c37f2808,poolID:----,device:disk,
shared:exclusive,propagateErrors:off,type:disk}
devices={index:2,iface:ide,shared:false,readonly:true,deviceId:8c3179ac-b322-4f5c-9449-c52e3665e0ae,address:{controller:0,target:0,unit:0,bus:1,type:drive},device:cdrom,path:,type:disk}
devices={device:cirrus,alias:video0,type:video,deviceId:a99468b6-02d4-4a77-8f94-e5df806030f6,address:{slot:0x02,bus:0x00,domain:0x,type:pci,function:0x0}}
devices={device:virtio-serial,type:controller,deviceId:b7580676-19fb-462f-a61e-677b65ad920a,address:{slot:0x03,bus:0x00,domain:0x,type:pci,function:0x0}}
devices={device:usb,type:controller,deviceId:c63092b3-7bd8-4b54-bcd3-51f34dce478a,address:{slot:0x01,bus:0x00,domain:0x,type:pci,function:0x2}}
devices={device:ide,type:controller,deviceId:c77c2c01-6ccc-404b-b8d6-5a7f0631a52f,address:{slot:0x01,bus:0x00,domain:0x,type:pci,function:0x1}}
As you can see there is no nic, and there is no nic in the qemu-kvm
command-line:
qemu 23290 1 14 00:23 ?01:44:26 /usr/libexec/qemu-kvm
-name HostedEngine -S -machine pc-i440fx-rhel7.2.0,accel=kvm,usb=off
-cpu qemu64,-svm -m 6144 -realtime mlock=off -s
mp 2,sockets=2,cores=1,threads=1 -uuid
497f5e4a-0c76-441a-b72e-724d7092d07e -smbios
type=1,manufacturer=oVirt,product=oVirt
Node,version=7-2.1511.el7.centos.2.10,serial=4C4C4544-004B-571
0-8044-B9C04F5A3732,uuid=497f5e4a-0c76-441a-b72e-724d7092d07e
-no-user-config -nodefaults -chardev
socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-HostedEngine/monitor.sock,serve
r,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc
base=2016-11-30T23:23:26,driftfix=slew -global
kvm-pit.lost_tick_policy=discard -no-hpet -no-reboot -boot strict=on
-device
piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device
virtio-serial-pci,id=virtio-serial0,max_ports=16,bus=pci.0,addr=0x3
-drive file=/var/run/vdsm/storage/2c3585cc-b7bc-4881-85b3-aa6514
991a26/d65b82e2-2ad1-4f4f-bfad-0277c37f2808/bb3218ba-cbe9-4cd0-b50b-931deae992f7,if=none,id=drive-virtio-disk0,format=raw,serial=d65b82e2-2ad1-4f4f-bfad-0277c37f2808,cache=none,werror=st
op,rerror=stop,aio=native -device
virtio-blk-pci,scsi=off,bus=pci.0,addr=0x6,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1
-drive if=none,id=drive-ide0-1-0,readonly=on,format=raw
-device ide-cd,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
-chardev
socket,id=charchannel0,path=/var/lib/libvirt/qemu/channels/497f5e4a-0c76-441a-b72e-724d7092d07e.com.redhat.rhevm
.vdsm,server,nowait -device
virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=com.redhat.rhevm.vdsm
-chardev socket,id=charchannel1,path=/var/lib/libvirt/qem
u/channels/497f5e4a-0c76-441a-b72e-724d7092d07e.org.qemu.guest_agent.0,server,nowait
-device
virtserialport,bus=virtio-serial0.0,nr=2,chardev=charchannel1,id=channel1,name=org.qemu.guest
_agent.0 -chardev
socket,id=charchannel2,path=/var/lib/libvirt/qemu/channels/497f5e4a-0c76-441a-b72e-724d7092d07e.org.ovirt.hosted-engine-setup.0,server,nowait
-device virtserialport,bus
=virtio-serial0.0,nr=3,chardev=charchannel2,id=channel2,name=org.ovirt.hosted-engine-setup.0
-vnc 0:0,password -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -msg
timestamp=on
I extracted the vm.conf from the storage domain and the nic is there:
mId=497f5e4a-0c76-441a-b72e-724d7092d07e
memSize=6144
display=vnc
devices={index:2,iface:ide,address:{ controller:0, target:0,unit:0,
bus:1,
type:drive},specParams:{},readonly:true,deviceId:857b98b3-cf43-4c2d-8061-e7f105234a65,path:,device:cdrom,shared
:false,type:disk}
devices={index:0,iface:virtio,format:raw,poolID:----,volumeID
Re: [ovirt-users] Help! My hosted engine lost his nic!
On Fri, Dec 2, 2016 at 5:03 PM, Cristian Mammoli wrote:
> Out of desperation i added the missing section int the OVF I extracted and
> "tarred" it back to the storage domain. The nic now is back...
>
Unfortunately I've to tell you that the engine will periodically rewrite
the OVF_STORE based on the configuration of the VM in its DB so in
principle also this one is just a temporary solution.
>
> I don't know if the query you suggested still make sense now... :/
> Anyway I don't have any nic in the gui even now
>
> [root@ovengine ~]# sudo -u postgres psql engine -c "select * from
> vm_device where type='interface' and vm_id='497f5e4a-0c76-441a-b72e
> -724d7092d07e'"
> could not change directory to "/root"
> device_id | vm_id | type
> | device | address|
> boot_order | spec_params | is_managed | is_plugged | is_readonly |
> _create_date | _update_date | alias | custom_properties | snaps
> hot_id | logical_name | is_using_scsi_reservation
> --+-
> -+---++-
> -++-
> +++-+---
> +--+---+---+--
> ---+--+---
> 6207e0d7-4dc9-406d-ab99-3facf45788f4 | 497f5e4a-0c76-441a-b72e-724d7092d07e
> | interface | bridge | {slot=0x04, bus=0x00, domain=0x, type=pci,
> function=0x0} | 0 | { } | f | t | f
> | 2016-12-02 01:46:42.999885+01 | | net0 |
>|
>| | f
> (1 row)
>
But this seams fine now.
Adding Roy on this.
>
> [root@ovengine ~]# sudo -u postgres psql engine -c "select * from vms
> where vm_guid='497f5e4a-0c76-441a-b72e-724d7092d07e'"
> could not change directory to "/root"
>vm_name| mem_size_mb | num_of_io_threads | nice_level | cpu_shares
> | vmt_guid | os | description |
> free_text_comment | vds_group_id | creation_date
> | auto_startup | is_stateless | is_smartcard_enabled |
> is_delete_protected | sso_method |
> dedicated_vm_for_vds | fail_back | default_boot_sequence | vm_type |
> vm_pool_spice_proxy | vds_group_name | transparent_hugepages |
> trusted_service | storage_pool_id| storage_pool_name
> | vds_group_description| vds_group_spice_proxy | vmt_name |
> vmt_mem_size_mb | vmt_os |
> vmt_creation_date| vmt_child_count | vmt_num_of_sockets |
> vmt_cpu_per_socket | vmt_threads_per_cpu | vmt_num_of_cpus |
> vmt_description | status | vm_ip | vm_ip_inet_array | vm_host
> | vm_pid | last_start_time | guest_cur_user_name | console_cur_user_name |
> guest_os | co
> nsole_user_id | guest_agent_nics_hash | run_on_vds |
> migrating_to_vds | app_list | vm_pool_name | vm_pool_id |
> vm_guid| num_of_monitors | single_qxl_pci |
> allow_console_reconnect | is_initialized | num_of_sockets | cpu_per_socket
> | threads_per_cpu | usb_policy | acpi_enable | session | num_of_cpus |
> quota_id | quota_name | quota_enforcement_type | kvm_enable | boot_sequence
> | utc_diff | last_
> vds_run_on | client_ip | guest_requested_memory | time_zone | cpu_user |
> cpu_sys | memory_usage_history |
> cpu_usage_history
> | network_usage_history | elapsed_time |
> usage_network_percent | disks_usage |
> usage_mem_percent | migration_progress_percent | usage_cpu_percent |
> run_on_vds_name |vds_group_cpu_name| de
> fault_display_type | priority | iso_path | origin |
> vds_group_compatibility_version | initrd_url | kernel_url | kernel_params
> | pause_status | exit_message | exit_status | migration_support |
> predefined_properties | userdefined_properties | min_allocated_mem |
> hash | cpu_pinning | db_generatio
> n | host_cpu_flags | tunnel_migration | vnc_keyboard_layout |
> is_run_and_pause | created_by_user_id | last_watchdog_event |
> last_watchdog_action | is_run_once | vm_fqdn | cpu_name |
> emulated_machine | current_cd | reason | exit_reason | instance_type_id |
> image_type_id | architecture | original_temp
> late_id | original_template_name | last_stop_time |
> migration_downtime | template_version_number | serial_number_policy |
> custom_serial_number | is_boot_menu_enabled | guest_cpu_count |
> next_run_config_exists | numatune_mode | is_spice_file_transfer_enabled |
> is_spice_copy_paste_enabled |
>cpu_profile_id| is_auto_converge | is_migrate_compressed |
> custom_emulated_machine | custom_cpu_name | spice_port | spice_tls_port |
> spice_ip | vnc_port | vnc_ip | guest_agent_status |
> guest_mem_buffered | guest_mem_cached | guest_mem_free |
>
Re: [ovirt-users] Help! My hosted engine lost his nic!
Out of desperation i added the missing section int the OVF I extracted
and "tarred" it back to the storage domain. The nic now is back...
I don't know if the query you suggested still make sense now... :/
Anyway I don't have any nic in the gui even now
[root@ovengine ~]# sudo -u postgres psql engine -c "select * from
vm_device where type='interface' and
vm_id='497f5e4a-0c76-441a-b72e-724d7092d07e'"
could not change directory to "/root"
device_id | vm_id |
type| device |
address| boot_order | spec_params |
is_managed | is_plugged | is_readonly | _create_date |
_update_date | alias | custom_properties | snaps
hot_id | logical_name | is_using_scsi_reservation
--+--+---++--++-+++-+---+--+---+---+--
---+--+---
6207e0d7-4dc9-406d-ab99-3facf45788f4 |
497f5e4a-0c76-441a-b72e-724d7092d07e | interface | bridge | {slot=0x04,
bus=0x00, domain=0x, type=pci, function=0x0} | 0 | {
} | f | t | f | 2016-12-02
01:46:42.999885+01 | | net0 | |
| | f
(1 row)
[root@ovengine ~]# sudo -u postgres psql engine -c "select * from
vms where vm_guid='497f5e4a-0c76-441a-b72e-724d7092d07e'"
could not change directory to "/root"
vm_name| mem_size_mb | num_of_io_threads | nice_level |
cpu_shares | vmt_guid | os | description |
free_text_comment | vds_group_id |
creation_date| auto_startup | is_stateless |
is_smartcard_enabled | is_delete_protected | sso_method |
dedicated_vm_for_vds | fail_back | default_boot_sequence | vm_type |
vm_pool_spice_proxy | vds_group_name | transparent_hugepages |
trusted_service | storage_pool_id|
storage_pool_name | vds_group_description|
vds_group_spice_proxy | vmt_name | vmt_mem_size_mb | vmt_os |
vmt_creation_date| vmt_child_count | vmt_num_of_sockets |
vmt_cpu_per_socket | vmt_threads_per_cpu | vmt_num_of_cpus |
vmt_description | status | vm_ip | vm_ip_inet_array |
vm_host | vm_pid | last_start_time | guest_cur_user_name |
console_cur_user_name | guest_os | co
nsole_user_id | guest_agent_nics_hash | run_on_vds |
migrating_to_vds | app_list | vm_pool_name | vm_pool_id |
vm_guid| num_of_monitors | single_qxl_pci |
allow_console_reconnect | is_initialized | num_of_sockets |
cpu_per_socket | threads_per_cpu | usb_policy | acpi_enable | session |
num_of_cpus | quota_id | quota_name | quota_enforcement_type |
kvm_enable | boot_sequence | utc_diff | last_
vds_run_on | client_ip | guest_requested_memory | time_zone | cpu_user |
cpu_sys | memory_usage_history
|cpu_usage_history
| network_usage_history | elapsed_time |
usage_network_percent | disks_usage |
usage_mem_percent | migration_progress_percent | usage_cpu_percent |
run_on_vds_name |vds_group_cpu_name| de
fault_display_type | priority | iso_path | origin |
vds_group_compatibility_version | initrd_url | kernel_url |
kernel_params | pause_status | exit_message | exit_status |
migration_support | predefined_properties | userdefined_properties |
min_allocated_mem | hash | cpu_pinning | db_generatio
n | host_cpu_flags | tunnel_migration | vnc_keyboard_layout |
is_run_and_pause | created_by_user_id | last_watchdog_event |
last_watchdog_action | is_run_once | vm_fqdn | cpu_name |
emulated_machine | current_cd | reason | exit_reason | instance_type_id
| image_type_id | architecture | original_temp
late_id | original_template_name | last_stop_time |
migration_downtime | template_version_number | serial_number_policy |
custom_serial_number | is_boot_menu_enabled | guest_cpu_count |
next_run_config_exists | numatune_mode | is_spice_file_transfer_enabled
| is_spice_copy_paste_enabled |
cpu_profile_id| is_auto_converge | is_migrate_compressed
| custom_emulated_machine | custom_cpu_name | spice_port |
spice_tls_port | spice_ip | vnc_port | vnc_ip |
guest_agent_status | guest_mem_buffered | guest_mem_cached |
guest_mem_free |small_icon_id |
large_icon_id | provider_id |
console_disconnect_action | guest_timezone_offset | guest_timezone_name
| guestos_arch | guestos_codename | guestos_distribution |
guestos_kernel_version | guestos_type | guestos_version
--+-+-
Re: [ovirt-users] Help! My hosted engine lost his nic!
On Thu, Dec 1, 2016 at 5:16 PM, Cristian Mammoli wrote:
> Here it is: http://cloud.apra.it/index.php/s/4cdcde8cafdb7a1c2c2374b02dc
> e118e
>
> I tarred all the agent.log on both servers.
>
> The engine was running on kvm01 and got shutdown on kvm01 around 10:35 AM
> on 29 November. But I think that's not the problem, it is supposed to shut
> down if the host can't reach the gateway. Probably the nic problem was
> already there but got triggered on reboot
>
> Btw I kept digging: I extracted the ovf from which vm.conf is generated:
>
> ovirt_hosted_engine_ha.lib.ovf.ovf_store.OVFStore::(getEngineVMOVF)
> OVF_STORE volume path: /rhev/data-center/mnt/blockSD/
> 2c3585cc-b7bc-4881-85b3-aa6514991a26/images/9c5e2121-f1a3-
> 4886-964c-c74fdfbbb3c1/ff765055-09c5-4b05-9cc7-5277b15c5d08
>
> # tar xvf /rhev/data-center/mnt/blockSD/2c3585cc-b7bc-4881-85b3-aa6514
> 991a26/images/9c5e2121-f1a3-4886-964c-c74fdfbbb3c1/ff76505
> 5-09c5-4b05-9cc7-5277b15c5d08
> 497f5e4a-0c76-441a-b72e-724d7092d07e.ovf
> info.json
>
> In the ovf file there is no Nic section...
>
>
Ciao Cristian,
do you see any interface for the engine VM in the engine admin portal?
Could you please execute this on the engine VM and share its output?
sudo -u postgres psql engine -c "select * from vm_device where
type='interface' and vm_id='497f5e4a-0c76-441a-b72e-724d7092d07e'"
sudo -u postgres psql engine -c "select * from vms where vm_guid='
497f5e4a-0c76-441a-b72e-724d7092d07e'"
thanks
> I uploaded the ovf on the same share as the logs
>
> Ty
>
>
> Il 01/12/2016 15:26, Yedidyah Bar David ha scritto:
>
>> On Thu, Dec 1, 2016 at 1:08 PM, Cristian Mammoli
>> wrote:
>>
>>> Hi, I upgraded an oVirt installation a month ago to the latest 3.6.7.
>>> Before
>>> it was 3.6.0 if I remember correctly.
>>> Everything went fine so far for a month or so.
>>>
>>> A couple of days ago the the default gateway got rebooted and the
>>> physical
>>> server hosting the HE decided to shut down the vm because it could not
>>> ping
>>> the gateway.
>>> The other host restarted the hevm but it now has *no nic*.
>>> As a workaround I attached a virtio nic via virsh but every time the vm
>>> gets
>>> restarted the nic get lost
>>>
>>> After a bit of troubleshooting and digging this is what I found:
>>>
>>> This is the /var/run/ovirt-hosted-engine-ha/vm.conf which, as far as I
>>> understand, gets extracted from the HE storage domain
>>>
>>> emulatedMachine=pc
>>> vmId=497f5e4a-0c76-441a-b72e-724d7092d07e
>>> smp=2
>>> memSize=6144
>>> spiceSecureChannels=smain,sdisplay,sinputs,scursor,splayback
>>> ,srecord,ssmartcard,susbredir
>>> vmName=HostedEngine
>>> display=vnc
>>> devices={index:0,iface:virtio,format:raw,bootOrder:1,address
>>> :{slot:0x06,bus:0x00,domain:0x,type:pci,function:0x0},
>>> volumeID:bb3218ba-cbe9-4cd0-b50b-931deae992f7,imageID:d65b82e2-2ad1-
>>> 4f4f-bfad-0277c37f2808,readonly:false,domainID:2c3585cc-
>>> b7bc-4881-85b3-aa6514991a26,deviceId:d65b82e2-2ad1-4f4f-
>>> bfad-0277c37f2808,poolID:----
>>> ,device:disk,
>>> shared:exclusive,propagateErrors:off,type:disk}
>>> devices={index:2,iface:ide,shared:false,readonly:true,device
>>> Id:8c3179ac-b322-4f5c-9449-c52e3665e0ae,address:{controll
>>> er:0,target:0,unit:0,bus:1,type:drive},device:cdrom,path:,type:disk}
>>> devices={device:cirrus,alias:video0,type:video,deviceId:a994
>>> 68b6-02d4-4a77-8f94-e5df806030f6,address:{slot:0x02,bus:
>>> 0x00,domain:0x,type:pci,function:0x0}}
>>> devices={device:virtio-serial,type:controller,deviceId:b7580
>>> 676-19fb-462f-a61e-677b65ad920a,address:{slot:0x03,bus:0x00,
>>> domain:0x,type:pci,function:0x0}}
>>> devices={device:usb,type:controller,deviceId:c63092b3-7bd8-
>>> 4b54-bcd3-51f34dce478a,address:{slot:0x01,bus:0x00,domain:
>>> 0x,type:pci,function:0x2}}
>>> devices={device:ide,type:controller,deviceId:c77c2c01-6ccc-
>>> 404b-b8d6-5a7f0631a52f,address:{slot:0x01,bus:0x00,domain:
>>> 0x,type:pci,function:0x1}}
>>>
>>> As you can see there is no nic, and there is no nic in the qemu-kvm
>>> command-line:
>>> qemu 23290 1 14 00:23 ?01:44:26 /usr/libexec/qemu-kvm
>>> -name
>>> HostedEngine -S -machine pc-i440fx-rhel7.2.0,accel=kvm,usb=off -cpu
>>> qemu64,-svm -m 6144 -realtime mlock=off -s
>>> mp 2,sockets=2,cores=1,threads=1 -uuid 497f5e4a-0c76-441a-b72e-724d70
>>> 92d07e
>>> -smbios type=1,manufacturer=oVirt,product=oVirt
>>> Node,version=7-2.1511.el7.centos.2.10,serial=4C4C4544-004B-571
>>> 0-8044-B9C04F5A3732,uuid=497f5e4a-0c76-441a-b72e-724d7092d07e
>>> -no-user-config -nodefaults -chardev
>>> socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-Host
>>> edEngine/monitor.sock,serve
>>> r,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc
>>> base=2016-11-30T23:23:26,driftfix=slew -global
>>> kvm-pit.lost_tick_policy=discard -no-hpet -no-reboot -boot strict=on
>>> -device
>>> piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device
>>> virtio-serial-pci,id=virtio-serial0,max_ports=16
Re: [ovirt-users] Help! My hosted engine lost his nic!
Here it is:
http://cloud.apra.it/index.php/s/4cdcde8cafdb7a1c2c2374b02dce118e
I tarred all the agent.log on both servers.
The engine was running on kvm01 and got shutdown on kvm01 around 10:35
AM on 29 November. But I think that's not the problem, it is supposed to
shut down if the host can't reach the gateway. Probably the nic problem
was already there but got triggered on reboot
Btw I kept digging: I extracted the ovf from which vm.conf is generated:
ovirt_hosted_engine_ha.lib.ovf.ovf_store.OVFStore::(getEngineVMOVF)
OVF_STORE volume path:
/rhev/data-center/mnt/blockSD/2c3585cc-b7bc-4881-85b3-aa6514991a26/images/9c5e2121-f1a3-4886-964c-c74fdfbbb3c1/ff765055-09c5-4b05-9cc7-5277b15c5d08
# tar xvf
/rhev/data-center/mnt/blockSD/2c3585cc-b7bc-4881-85b3-aa6514991a26/images/9c5e2121-f1a3-4886-964c-c74fdfbbb3c1/ff765055-09c5-4b05-9cc7-5277b15c5d08
497f5e4a-0c76-441a-b72e-724d7092d07e.ovf
info.json
In the ovf file there is no Nic section...
I uploaded the ovf on the same share as the logs
Ty
Il 01/12/2016 15:26, Yedidyah Bar David ha scritto:
On Thu, Dec 1, 2016 at 1:08 PM, Cristian Mammoli wrote:
Hi, I upgraded an oVirt installation a month ago to the latest 3.6.7. Before
it was 3.6.0 if I remember correctly.
Everything went fine so far for a month or so.
A couple of days ago the the default gateway got rebooted and the physical
server hosting the HE decided to shut down the vm because it could not ping
the gateway.
The other host restarted the hevm but it now has *no nic*.
As a workaround I attached a virtio nic via virsh but every time the vm gets
restarted the nic get lost
After a bit of troubleshooting and digging this is what I found:
This is the /var/run/ovirt-hosted-engine-ha/vm.conf which, as far as I
understand, gets extracted from the HE storage domain
emulatedMachine=pc
vmId=497f5e4a-0c76-441a-b72e-724d7092d07e
smp=2
memSize=6144
spiceSecureChannels=smain,sdisplay,sinputs,scursor,splayback,srecord,ssmartcard,susbredir
vmName=HostedEngine
display=vnc
devices={index:0,iface:virtio,format:raw,bootOrder:1,address:{slot:0x06,bus:0x00,domain:0x,type:pci,function:0x0},volumeID:bb3218ba-cbe9-4cd0-b50b-931deae992f7,imageID:d65b82e2-2ad1-
4f4f-bfad-0277c37f2808,readonly:false,domainID:2c3585cc-b7bc-4881-85b3-aa6514991a26,deviceId:d65b82e2-2ad1-4f4f-bfad-0277c37f2808,poolID:----,device:disk,
shared:exclusive,propagateErrors:off,type:disk}
devices={index:2,iface:ide,shared:false,readonly:true,deviceId:8c3179ac-b322-4f5c-9449-c52e3665e0ae,address:{controller:0,target:0,unit:0,bus:1,type:drive},device:cdrom,path:,type:disk}
devices={device:cirrus,alias:video0,type:video,deviceId:a99468b6-02d4-4a77-8f94-e5df806030f6,address:{slot:0x02,bus:0x00,domain:0x,type:pci,function:0x0}}
devices={device:virtio-serial,type:controller,deviceId:b7580676-19fb-462f-a61e-677b65ad920a,address:{slot:0x03,bus:0x00,domain:0x,type:pci,function:0x0}}
devices={device:usb,type:controller,deviceId:c63092b3-7bd8-4b54-bcd3-51f34dce478a,address:{slot:0x01,bus:0x00,domain:0x,type:pci,function:0x2}}
devices={device:ide,type:controller,deviceId:c77c2c01-6ccc-404b-b8d6-5a7f0631a52f,address:{slot:0x01,bus:0x00,domain:0x,type:pci,function:0x1}}
As you can see there is no nic, and there is no nic in the qemu-kvm
command-line:
qemu 23290 1 14 00:23 ?01:44:26 /usr/libexec/qemu-kvm -name
HostedEngine -S -machine pc-i440fx-rhel7.2.0,accel=kvm,usb=off -cpu
qemu64,-svm -m 6144 -realtime mlock=off -s
mp 2,sockets=2,cores=1,threads=1 -uuid 497f5e4a-0c76-441a-b72e-724d7092d07e
-smbios type=1,manufacturer=oVirt,product=oVirt
Node,version=7-2.1511.el7.centos.2.10,serial=4C4C4544-004B-571
0-8044-B9C04F5A3732,uuid=497f5e4a-0c76-441a-b72e-724d7092d07e
-no-user-config -nodefaults -chardev
socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-HostedEngine/monitor.sock,serve
r,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc
base=2016-11-30T23:23:26,driftfix=slew -global
kvm-pit.lost_tick_policy=discard -no-hpet -no-reboot -boot strict=on -device
piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device
virtio-serial-pci,id=virtio-serial0,max_ports=16,bus=pci.0,addr=0x3 -drive
file=/var/run/vdsm/storage/2c3585cc-b7bc-4881-85b3-aa6514
991a26/d65b82e2-2ad1-4f4f-bfad-0277c37f2808/bb3218ba-cbe9-4cd0-b50b-931deae992f7,if=none,id=drive-virtio-disk0,format=raw,serial=d65b82e2-2ad1-4f4f-bfad-0277c37f2808,cache=none,werror=st
op,rerror=stop,aio=native -device
virtio-blk-pci,scsi=off,bus=pci.0,addr=0x6,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1
-drive if=none,id=drive-ide0-1-0,readonly=on,format=raw
-device ide-cd,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0 -chardev
socket,id=charchannel0,path=/var/lib/libvirt/qemu/channels/497f5e4a-0c76-441a-b72e-724d7092d07e.com.redhat.rhevm
.vdsm,server,nowait -device
virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=com.redhat.rhevm.vdsm
-chardev socket,id=charchannel1,path=/var/lib/libvirt/
Re: [ovirt-users] Help! My hosted engine lost his nic!
On Thu, Dec 1, 2016 at 1:08 PM, Cristian Mammoli wrote:
> Hi, I upgraded an oVirt installation a month ago to the latest 3.6.7. Before
> it was 3.6.0 if I remember correctly.
> Everything went fine so far for a month or so.
>
> A couple of days ago the the default gateway got rebooted and the physical
> server hosting the HE decided to shut down the vm because it could not ping
> the gateway.
> The other host restarted the hevm but it now has *no nic*.
> As a workaround I attached a virtio nic via virsh but every time the vm gets
> restarted the nic get lost
>
> After a bit of troubleshooting and digging this is what I found:
>
> This is the /var/run/ovirt-hosted-engine-ha/vm.conf which, as far as I
> understand, gets extracted from the HE storage domain
>
> emulatedMachine=pc
> vmId=497f5e4a-0c76-441a-b72e-724d7092d07e
> smp=2
> memSize=6144
> spiceSecureChannels=smain,sdisplay,sinputs,scursor,splayback,srecord,ssmartcard,susbredir
> vmName=HostedEngine
> display=vnc
> devices={index:0,iface:virtio,format:raw,bootOrder:1,address:{slot:0x06,bus:0x00,domain:0x,type:pci,function:0x0},volumeID:bb3218ba-cbe9-4cd0-b50b-931deae992f7,imageID:d65b82e2-2ad1-
> 4f4f-bfad-0277c37f2808,readonly:false,domainID:2c3585cc-b7bc-4881-85b3-aa6514991a26,deviceId:d65b82e2-2ad1-4f4f-bfad-0277c37f2808,poolID:----,device:disk,
> shared:exclusive,propagateErrors:off,type:disk}
> devices={index:2,iface:ide,shared:false,readonly:true,deviceId:8c3179ac-b322-4f5c-9449-c52e3665e0ae,address:{controller:0,target:0,unit:0,bus:1,type:drive},device:cdrom,path:,type:disk}
> devices={device:cirrus,alias:video0,type:video,deviceId:a99468b6-02d4-4a77-8f94-e5df806030f6,address:{slot:0x02,bus:0x00,domain:0x,type:pci,function:0x0}}
> devices={device:virtio-serial,type:controller,deviceId:b7580676-19fb-462f-a61e-677b65ad920a,address:{slot:0x03,bus:0x00,domain:0x,type:pci,function:0x0}}
> devices={device:usb,type:controller,deviceId:c63092b3-7bd8-4b54-bcd3-51f34dce478a,address:{slot:0x01,bus:0x00,domain:0x,type:pci,function:0x2}}
> devices={device:ide,type:controller,deviceId:c77c2c01-6ccc-404b-b8d6-5a7f0631a52f,address:{slot:0x01,bus:0x00,domain:0x,type:pci,function:0x1}}
>
> As you can see there is no nic, and there is no nic in the qemu-kvm
> command-line:
> qemu 23290 1 14 00:23 ?01:44:26 /usr/libexec/qemu-kvm -name
> HostedEngine -S -machine pc-i440fx-rhel7.2.0,accel=kvm,usb=off -cpu
> qemu64,-svm -m 6144 -realtime mlock=off -s
> mp 2,sockets=2,cores=1,threads=1 -uuid 497f5e4a-0c76-441a-b72e-724d7092d07e
> -smbios type=1,manufacturer=oVirt,product=oVirt
> Node,version=7-2.1511.el7.centos.2.10,serial=4C4C4544-004B-571
> 0-8044-B9C04F5A3732,uuid=497f5e4a-0c76-441a-b72e-724d7092d07e
> -no-user-config -nodefaults -chardev
> socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-HostedEngine/monitor.sock,serve
> r,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc
> base=2016-11-30T23:23:26,driftfix=slew -global
> kvm-pit.lost_tick_policy=discard -no-hpet -no-reboot -boot strict=on -device
> piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device
> virtio-serial-pci,id=virtio-serial0,max_ports=16,bus=pci.0,addr=0x3 -drive
> file=/var/run/vdsm/storage/2c3585cc-b7bc-4881-85b3-aa6514
> 991a26/d65b82e2-2ad1-4f4f-bfad-0277c37f2808/bb3218ba-cbe9-4cd0-b50b-931deae992f7,if=none,id=drive-virtio-disk0,format=raw,serial=d65b82e2-2ad1-4f4f-bfad-0277c37f2808,cache=none,werror=st
> op,rerror=stop,aio=native -device
> virtio-blk-pci,scsi=off,bus=pci.0,addr=0x6,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1
> -drive if=none,id=drive-ide0-1-0,readonly=on,format=raw
> -device ide-cd,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0 -chardev
> socket,id=charchannel0,path=/var/lib/libvirt/qemu/channels/497f5e4a-0c76-441a-b72e-724d7092d07e.com.redhat.rhevm
> .vdsm,server,nowait -device
> virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=com.redhat.rhevm.vdsm
> -chardev socket,id=charchannel1,path=/var/lib/libvirt/qem
> u/channels/497f5e4a-0c76-441a-b72e-724d7092d07e.org.qemu.guest_agent.0,server,nowait
> -device
> virtserialport,bus=virtio-serial0.0,nr=2,chardev=charchannel1,id=channel1,name=org.qemu.guest
> _agent.0 -chardev
> socket,id=charchannel2,path=/var/lib/libvirt/qemu/channels/497f5e4a-0c76-441a-b72e-724d7092d07e.org.ovirt.hosted-engine-setup.0,server,nowait
> -device virtserialport,bus
> =virtio-serial0.0,nr=3,chardev=charchannel2,id=channel2,name=org.ovirt.hosted-engine-setup.0
> -vnc 0:0,password -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -msg
> timestamp=on
>
> I extracted the vm.conf from the storage domain and the nic is there:
> mId=497f5e4a-0c76-441a-b72e-724d7092d07e
> memSize=6144
> display=vnc
> devices={index:2,iface:ide,address:{ controller:0, target:0,unit:0, bus:1,
> type:drive},specParams:{},readonly:true,deviceId:857b98b3-cf43-4c2d-8061-e7f105234a65,path:,device:cdrom,shared
> :false,type:disk}
> devices={index:0,iface:virtio,fo
[ovirt-users] Help! My hosted engine lost his nic!
Hi, I upgraded an oVirt installation a month ago to the latest 3.6.7.
Before it was 3.6.0 if I remember correctly.
Everything went fine so far for a month or so.
A couple of days ago the the default gateway got rebooted and the
physical server hosting the HE decided to shut down the vm because it
could not ping the gateway.
The other host restarted the hevm but it now has *no nic*.
As a workaround I attached a virtio nic via virsh but every time the vm
gets restarted the nic get lost
After a bit of troubleshooting and digging this is what I found:
This is the /var/run/ovirt-hosted-engine-ha/vm.conf which, as far as I
understand, gets extracted from the HE storage domain
emulatedMachine=pc
vmId=497f5e4a-0c76-441a-b72e-724d7092d07e
smp=2
memSize=6144
spiceSecureChannels=smain,sdisplay,sinputs,scursor,splayback,srecord,ssmartcard,susbredir
vmName=HostedEngine
display=vnc
devices={index:0,iface:virtio,format:raw,bootOrder:1,address:{slot:0x06,bus:0x00,domain:0x,type:pci,function:0x0},volumeID:bb3218ba-cbe9-4cd0-b50b-931deae992f7,imageID:d65b82e2-2ad1-
4f4f-bfad-0277c37f2808,readonly:false,domainID:2c3585cc-b7bc-4881-85b3-aa6514991a26,deviceId:d65b82e2-2ad1-4f4f-bfad-0277c37f2808,poolID:----,device:disk,
shared:exclusive,propagateErrors:off,type:disk}
devices={index:2,iface:ide,shared:false,readonly:true,deviceId:8c3179ac-b322-4f5c-9449-c52e3665e0ae,address:{controller:0,target:0,unit:0,bus:1,type:drive},device:cdrom,path:,type:disk}
devices={device:cirrus,alias:video0,type:video,deviceId:a99468b6-02d4-4a77-8f94-e5df806030f6,address:{slot:0x02,bus:0x00,domain:0x,type:pci,function:0x0}}
devices={device:virtio-serial,type:controller,deviceId:b7580676-19fb-462f-a61e-677b65ad920a,address:{slot:0x03,bus:0x00,domain:0x,type:pci,function:0x0}}
devices={device:usb,type:controller,deviceId:c63092b3-7bd8-4b54-bcd3-51f34dce478a,address:{slot:0x01,bus:0x00,domain:0x,type:pci,function:0x2}}
devices={device:ide,type:controller,deviceId:c77c2c01-6ccc-404b-b8d6-5a7f0631a52f,address:{slot:0x01,bus:0x00,domain:0x,type:pci,function:0x1}}
As you can see there is no nic, and there is no nic in the qemu-kvm
command-line:
qemu 23290 1 14 00:23 ?01:44:26 /usr/libexec/qemu-kvm
-name HostedEngine -S -machine pc-i440fx-rhel7.2.0,accel=kvm,usb=off
-cpu qemu64,-svm -m 6144 -realtime mlock=off -s
mp 2,sockets=2,cores=1,threads=1 -uuid
497f5e4a-0c76-441a-b72e-724d7092d07e -smbios
type=1,manufacturer=oVirt,product=oVirt
Node,version=7-2.1511.el7.centos.2.10,serial=4C4C4544-004B-571
0-8044-B9C04F5A3732,uuid=497f5e4a-0c76-441a-b72e-724d7092d07e
-no-user-config -nodefaults -chardev
socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-HostedEngine/monitor.sock,serve
r,nowait -mon chardev=charmonitor,id=monitor,mode=control -rtc
base=2016-11-30T23:23:26,driftfix=slew -global
kvm-pit.lost_tick_policy=discard -no-hpet -no-reboot -boot strict=on -device
piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2 -device
virtio-serial-pci,id=virtio-serial0,max_ports=16,bus=pci.0,addr=0x3
-drive file=/var/run/vdsm/storage/2c3585cc-b7bc-4881-85b3-aa6514
991a26/d65b82e2-2ad1-4f4f-bfad-0277c37f2808/bb3218ba-cbe9-4cd0-b50b-931deae992f7,if=none,id=drive-virtio-disk0,format=raw,serial=d65b82e2-2ad1-4f4f-bfad-0277c37f2808,cache=none,werror=st
op,rerror=stop,aio=native -device
virtio-blk-pci,scsi=off,bus=pci.0,addr=0x6,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1
-drive if=none,id=drive-ide0-1-0,readonly=on,format=raw
-device ide-cd,bus=ide.1,unit=0,drive=drive-ide0-1-0,id=ide0-1-0
-chardev
socket,id=charchannel0,path=/var/lib/libvirt/qemu/channels/497f5e4a-0c76-441a-b72e-724d7092d07e.com.redhat.rhevm
.vdsm,server,nowait -device
virtserialport,bus=virtio-serial0.0,nr=1,chardev=charchannel0,id=channel0,name=com.redhat.rhevm.vdsm
-chardev socket,id=charchannel1,path=/var/lib/libvirt/qem
u/channels/497f5e4a-0c76-441a-b72e-724d7092d07e.org.qemu.guest_agent.0,server,nowait
-device
virtserialport,bus=virtio-serial0.0,nr=2,chardev=charchannel1,id=channel1,name=org.qemu.guest
_agent.0 -chardev
socket,id=charchannel2,path=/var/lib/libvirt/qemu/channels/497f5e4a-0c76-441a-b72e-724d7092d07e.org.ovirt.hosted-engine-setup.0,server,nowait
-device virtserialport,bus
=virtio-serial0.0,nr=3,chardev=charchannel2,id=channel2,name=org.ovirt.hosted-engine-setup.0
-vnc 0:0,password -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -msg
timestamp=on
I extracted the vm.conf from the storage domain and the nic is there:
mId=497f5e4a-0c76-441a-b72e-724d7092d07e
memSize=6144
display=vnc
devices={index:2,iface:ide,address:{ controller:0, target:0,unit:0,
bus:1,
type:drive},specParams:{},readonly:true,deviceId:857b98b3-cf43-4c2d-8061-e7f105234a65,path:,device:cdrom,shared
:false,type:disk}
devices={index:0,iface:virtio,format:raw,poolID:----,volumeID:bb3218ba-cbe9-4cd0-b50b-931deae992f7,imageID:d65b82e2-2ad1-4f4f-bfad-0277c37f2808,specParams
:{},readon
