Hi!

The broker error is:

==> /var/log/ovirt-hosted-engine-ha/agent.log <==
MainThread::INFO::2016-06-27 09:27:03,311::brokerlink::140::ovirt_hosted_engine_ha.lib.brokerlink.BrokerLink::(start_monitor) Success, id 140293563619152

==> /var/log/ovirt-hosted-engine-ha/broker.log <==
Thread-25::ERROR::2016-06-27 09:27:03,314::listener::182::ovirt_hosted_engine_ha.broker.listener.ConnectionHandler::(handle) Error while serving connection
Traceback (most recent call last):
File "/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/broker/listener.py", line 166, in handle
    data)
File "/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/broker/listener.py", line 299, in _dispatch
    .set_storage_domain(client, sd_type, **options)
File "/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/broker/storage_broker.py", line 66, in set_storage_domain
    self._backends[client].connect()
File "/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/lib/storage_backends.py", line 400, in connect
    volUUID=volume.volume_uuid
File "/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/lib/storage_backends.py", line 245, in _get_volume_path
    volUUID
  File "/usr/lib64/python2.7/xmlrpclib.py", line 1233, in __call__
    return self.__send(self.__name, args)
  File "/usr/lib64/python2.7/xmlrpclib.py", line 1587, in __request
    verbose=self.__verbose
  File "/usr/lib64/python2.7/xmlrpclib.py", line 1273, in request
    return self.single_request(host, handler, request_body, verbose)
  File "/usr/lib64/python2.7/xmlrpclib.py", line 1301, in single_request
    self.send_content(h, request_body)
  File "/usr/lib64/python2.7/xmlrpclib.py", line 1448, in send_content
    connection.endheaders(request_body)
  File "/usr/lib64/python2.7/httplib.py", line 975, in endheaders
    self._send_output(message_body)
  File "/usr/lib64/python2.7/httplib.py", line 835, in _send_output
    self.send(msg)
  File "/usr/lib64/python2.7/httplib.py", line 797, in send
    self.connect()
File "/usr/lib/python2.7/site-packages/vdsm/m2cutils.py", line 203, in connect
    sock = socket.create_connection((self.host, self.port), self.timeout)
  File "/usr/lib64/python2.7/socket.py", line 571, in create_connection
    raise err
error: [Errno 101] Network is unreachable


Broker PID is:

[root@ovirt01 ~]# ps -ef | grep bro
vdsm 925 1 0 09:23 ? 00:00:01 /usr/bin/python /usr/share/ovirt-hosted-engine-ha/ovirt-ha-broker --no-daemon
root      3558  3405  0 09:28 pts/0    00:00:00 grep --color=auto bro


[root@ovirt01 ~]# strace -f -p 925
[..]
[pid  3748] munmap(0x7f98bded0000, 4096) = 0
_*[pid  3748] socket(PF_INET6, SOCK_STREAM, IPPROTO_TCP) = 8*__*
*__*[pid 3748] connect(8, {sa_family=AF_INET6, sin6_port=htons(54321), inet_pton(AF_INET6, "::", &sin6_addr), sin6_flowinfo=0, sin6_scope_id=0}, 28) = -1 ENETUNREACH (Network is unreachable)*_
[pid  3748] close(8)                    = 0
[pid  3748] select(0, NULL, NULL, NULL, {1, 0} <unfinished ...>
[pid 3742] <... recvfrom resumed> "set-storage-domain VdsmBackend h"..., 8192, 0, NULL, NULL) = 676
[pid  3742] fcntl(6, F_GETFL)           = 0x2 (flags O_RDWR)
[pid  3742] fcntl(6, F_SETFL, O_RDWR|O_NONBLOCK) = 0
[pid 3742] openat(AT_FDCWD, "/rhev/data-center/mnt", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 8
[pid  3742] getdents(8, /* 7 entries */, 32768) = 312
[pid  3742] getdents(8, /* 0 entries */, 32768) = 0
[pid  3742] close(8)                    = 0
[pid 3742] access("/rhev/data-center/mnt/ovirt01.hawai.lan:_home_storage-ovirtShe/46f55a31-f35f-465c-b3e2-df45c05e06a7", F_OK) = -1 ENOENT (No such file or directory) [pid 3742] access("/rhev/data-center/mnt/glusterSD/46f55a31-f35f-465c-b3e2-df45c05e06a7", F_OK) = -1 ENOENT (No such file or directory) [pid 3742] access("/rhev/data-center/mnt/ovirtbk-mount.hawai.lan:_engine/46f55a31-f35f-465c-b3e2-df45c05e06a7", F_OK) = -1 ENOENT (No such file or directory) [pid 3742] access("/rhev/data-center/mnt/ovirtbk-sheng.hawai.lan:_var_lib_exports_iso/46f55a31-f35f-465c-b3e2-df45c05e06a7", F_OK) = -1 ENOENT (No such file or directory) [pid 3742] access("/rhev/data-center/mnt/ovirt01.hawai.lan:_engine/46f55a31-f35f-465c-b3e2-df45c05e06a7", F_OK) = 0
[pid  3742] access("/etc/pki/vdsm/keys/vdsmkey.pem", R_OK) = 0
[pid  3742] access("/etc/pki/vdsm/certs/vdsmcert.pem", R_OK) = 0
[pid  3742] access("/etc/pki/vdsm/certs/cacert.pem", R_OK) = 0
[pid  3742] open("/etc/pki/vdsm/certs/vdsmcert.pem", O_RDONLY) = 8
[pid  3742] fstat(8, {st_mode=S_IFREG|0644, st_size=1574, ...}) = 0
[pid 3742] mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f98bded0000
[pid  3742] read(8, "-----BEGIN CERTIFICATE-----\nMIIE"..., 4096) = 1574
[pid  3742] read(8, "", 4096)           = 0
[pid  3742] close(8)                    = 0
[pid  3742] munmap(0x7f98bded0000, 4096) = 0
[pid  3742] open("/etc/pki/vdsm/keys/vdsmkey.pem", O_RDONLY) = 8
[pid  3742] fstat(8, {st_mode=S_IFREG|0440, st_size=1675, ...}) = 0
[pid 3742] mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f98bded0000
[pid  3742] read(8, "-----BEGIN RSA PRIVATE KEY-----\n"..., 4096) = 1675
[pid  3742] close(8)                    = 0
[pid  3742] munmap(0x7f98bded0000, 4096) = 0
[*_pid  3742] socket(PF_INET6, SOCK_STREAM, IPPROTO_TCP) = 8_**_
_**_[pid 3742] connect(8, {sa_family=AF_INET6, sin6_port=htons(54321), inet_pton(AF_INET6, "::", &sin6_addr), sin6_flowinfo=0, sin6_scope_id=0}, 28) = -1 ENETUNREACH (Network is unreachable)_**_
_**_[pid  3742] close(8)                    = 0_*
[pid 3742] stat("/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/broker/listener.py", {st_mode=S_IFREG|0644, st_size=14078, ...}) = 0 [pid 3742] stat("/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/broker/listener.py", {st_mode=S_IFREG|0644, st_size=14078, ...}) = 0 [pid 3742] stat("/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/broker/storage_broker.py", {st_mode=S_IFREG|0644, st_size=8885, ...}) = 0 [pid 3742] stat("/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/lib/storage_backends.py", {st_mode=S_IFREG|0644, st_size=32723, ...}) = 0 [pid 3742] stat("/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/lib/storage_backends.py", {st_mode=S_IFREG|0644, st_size=32723, ...}) = 0 [pid 3742] stat("/usr/lib64/python2.7/xmlrpclib.py", {st_mode=S_IFREG|0644, st_size=51801, ...}) = 0 [pid 3742] stat("/usr/lib64/python2.7/xmlrpclib.py", {st_mode=S_IFREG|0644, st_size=51801, ...}) = 0 [pid 3742] stat("/usr/lib64/python2.7/xmlrpclib.py", {st_mode=S_IFREG|0644, st_size=51801, ...}) = 0 [pid 3742] stat("/usr/lib64/python2.7/xmlrpclib.py", {st_mode=S_IFREG|0644, st_size=51801, ...}) = 0 [pid 3742] stat("/usr/lib64/python2.7/xmlrpclib.py", {st_mode=S_IFREG|0644, st_size=51801, ...}) = 0 [pid 3742] stat("/usr/lib64/python2.7/httplib.py", {st_mode=S_IFREG|0644, st_size=48234, ...}) = 0 [pid 3742] stat("/usr/lib64/python2.7/httplib.py", {st_mode=S_IFREG|0644, st_size=48234, ...}) = 0 [pid 3742] stat("/usr/lib64/python2.7/httplib.py", {st_mode=S_IFREG|0644, st_size=48234, ...}) = 0 [pid 3742] stat("/usr/lib/python2.7/site-packages/vdsm/m2cutils.py", {st_mode=S_IFREG|0644, st_size=10720, ...}) = 0 [pid 3742] stat("/usr/lib64/python2.7/socket.py", {st_mode=S_IFREG|0644, st_size=20512, ...}) = 0 [pid 3742] sendto(3, "<11>ovirt-ha-broker ovirt_hosted"..., 1970, 0, NULL, 0) = 1970 [pid 3742] stat("/etc/localtime", {st_mode=S_IFREG|0644, st_size=2652, ...}) = 0
[pid  3742] write(4, "Thread-49::ERROR::2016-06-27 09:"..., 2012) = 2012
[pid  3742] write(2, "ERROR:ovirt_hosted_engine_ha.bro"..., 1950) = 1950
[pid 3742] poll([{fd=6, events=POLLOUT}], 1, 700) = 1 ([{fd=6, revents=POLLOUT}]) [pid 3742] sendto(6, "failure <class 'socket.error'>\n", 31, 0, NULL, 0) = 31
[pid  3742] fcntl(6, F_GETFL)           = 0x802 (flags O_RDWR|O_NONBLOCK)
[pid  3742] fcntl(6, F_SETFL, O_RDWR)   = 0
[pid  3742] recvfrom(6,  <unfinished ...>

enabling IPV6 only in ovirtmgmt interface the broker still report python exception, but in vdsm log I see:

jsonrpc.Executor/4::DEBUG::2016-06-27 10:01:36,697::__init__::550::jsonrpc.JsonRpcServer::(_serveRequest) Return 'Host.getHardwareInfo' in bridge with {'systemProductName': 'To be filled by O.E.M.', 'systemSerialNumber': 'To be filled by O.E.M.', 'systemFamily': 'To be filled by O.E.M.', 'systemVersion': 'To be filled by O.E.M.', 'systemUUID': 'F90B3100-D83F-11DD-8DD8-40167E3684F1', 'systemManufacturer': 'To be filled by O.E.M.'} Reactor thread::INFO::2016-06-27 10:01:38,703::protocoldetector::72::ProtocolDetector.AcceptorImpl::(handle_accept) *Accepting connection from ::ffff:192.168.1.50:56228* Reactor thread::DEBUG::2016-06-27 10:01:38,713::protocoldetector::85::ProtocolDetector.Detector::(__init__) Using required_size=11 Reactor thread::INFO::2016-06-27 10:01:38,714::protocoldetector::121::ProtocolDetector.Detector::(handle_read) Detected protocol stomp from ::ffff:192.168.1.50:56228 Reactor thread::INFO::2016-06-27 10:01:38,714::stompreactor::101::Broker.StompAdapter::(_cmd_connect) Processing CONNECT request JsonRpc (StompReactor)::INFO::2016-06-27 10:01:38,715::stompreactor::128::Broker.StompAdapter::(_cmd_subscribe) Subscribe command received Reactor thread::DEBUG::2016-06-27 10:01:38,716::stompreactor::482::protocoldetector.StompDetector::(handle_socket) Stomp detected from ('::ffff:192.168.1.50', 56228) jsonrpc.Executor/5::DEBUG::2016-06-27 10:01:40,319::__init__::522::jsonrpc.JsonRpcServer::(_serveRequest) Calling 'Host.getHardwareInfo' in bridge with {}




Il 26/06/2016 10.30, Dan Kenigsberg ha scritto:
On Fri, Jun 24, 2016 at 06:45:24PM +0200, Stefano Danzi wrote:
HI!!!!!!

I found a workaround!!!!!!!!!!!!!!!!

the brocker process try to connect to vdsm to IPV4 host address using an
IPV6 connection
(I noticed that doing a strace to the process),
but ipv6 is not intialized at boot. (why connect to IPV4 address using
IPV6?)
Acutally, we take an effort to disable ipv6 on ovirt host networks.
Keeping them open without explicit request was deemed a security issue.

Can you share your strace line and the relevant lines in vdsm.log? I
don't understand what is the issue that you are reporting.

I added the following lines to crontab:

@reboot echo 'echo 0 > /proc/sys/net/ipv6/conf/lo/disable_ipv6' |
/usr/bin/at now+1 minutes
@reboot echo 'echo 0 > /proc/sys/net/ipv6/conf/ovirtmgmt/disable_ipv6' |
/usr/bin/at now+1 minutes
@reboot echo '/usr/sbin/route add default gw 192.168.1.254'  | /usr/bin/at
now+1 minutes



Il 24/06/2016 12.36, Stefano Danzi ha scritto:
How I can change self hosted engine configuration to mount directly
gluster storage without pass through gluster NFS?

Maybe this solve....

Il 24/06/2016 10.16, Stefano Danzi ha scritto:
After an additional yum clean all && yum update was updated some
other rpms.

Something changed.....
My setup has engine storage on gluster, but mounted with NFS.
Now gluster daemon don't automatically start at boot. After starting
manually gluster the error is the same:

==> /var/log/ovirt-hosted-engine-ha/broker.log <==
Thread-19::ERROR::2016-06-24 
10:10:36,758::listener::182::ovirt_hosted_engine_ha.broker.listener.ConnectionHandler::(handle)
Error while serving connection
Traceback (most recent call last):
   File 
"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/broker/listener.py",
line 166, in handle
     data)
   File 
"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/broker/listener.py",
line 299, in _dispatch
     .set_storage_domain(client, sd_type, **options)
   File 
"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/broker/storage_broker.py",
line 66, in set_storage_domain
     self._backends[client].connect()
   File 
"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/lib/storage_backends.py",
line 400, in connect
     volUUID=volume.volume_uuid
   File 
"/usr/lib/python2.7/site-packages/ovirt_hosted_engine_ha/lib/storage_backends.py",
line 245, in _get_volume_path
     volUUID
   File "/usr/lib64/python2.7/xmlrpclib.py", line 1233, in __call__
     return self.__send(self.__name, args)
   File "/usr/lib64/python2.7/xmlrpclib.py", line 1587, in __request
     verbose=self.__verbose
   File "/usr/lib64/python2.7/xmlrpclib.py", line 1273, in request
     return self.single_request(host, handler, request_body, verbose)
   File "/usr/lib64/python2.7/xmlrpclib.py", line 1301, in single_request
     self.send_content(h, request_body)
   File "/usr/lib64/python2.7/xmlrpclib.py", line 1448, in send_content
     connection.endheaders(request_body)
   File "/usr/lib64/python2.7/httplib.py", line 975, in endheaders
     self._send_output(message_body)
   File "/usr/lib64/python2.7/httplib.py", line 835, in _send_output
     self.send(msg)
   File "/usr/lib64/python2.7/httplib.py", line 797, in send
     self.connect()
   File "/usr/lib/python2.7/site-packages/vdsm/m2cutils.py", line
203, in connect
     sock = socket.create_connection((self.host, self.port), self.timeout)
   File "/usr/lib64/python2.7/socket.py", line 571, in create_connection
     raise err
error: [Errno 101] Network is unreachable


VDSM.log

jsonrpc.Executor/5::DEBUG::2016-06-24
10:10:21,694::task::995::Storage.TaskManager.Task::(_decref)
Task=`5c3b6f30-d3a8-431e-9dd0-8df79b171709`::ref 0
aborting False
jsonrpc.Executor/5::WARNING::2016-06-24
10:10:21,694::vdsmapi::143::SchemaCache::(_report_inconsistency)
Following parameters ['type'] were not recogn
ized
jsonrpc.Executor/5::WARNING::2016-06-24
10:10:21,694::vdsmapi::143::SchemaCache::(_report_inconsistency)
Provided value "2" not defined in DiskType en
um for Volume.getInfo
jsonrpc.Executor/5::WARNING::2016-06-24
10:10:21,694::vdsmapi::143::SchemaCache::(_report_inconsistency)
Parameter capacity is not uint type
jsonrpc.Executor/5::WARNING::2016-06-24
10:10:21,694::vdsmapi::143::SchemaCache::(_report_inconsistency)
Required property allocType is not provided w
hen calling Volume.getInfo
jsonrpc.Executor/5::WARNING::2016-06-24
10:10:21,694::vdsmapi::143::SchemaCache::(_report_inconsistency)
Parameter mtime is not uint type
jsonrpc.Executor/5::WARNING::2016-06-24
10:10:21,694::vdsmapi::143::SchemaCache::(_report_inconsistency)
Parameter ctime is not int type
jsonrpc.Executor/5::WARNING::2016-06-24
10:10:21,695::vdsmapi::143::SchemaCache::(_report_inconsistency)
Parameter truesize is not uint type
jsonrpc.Executor/5::WARNING::2016-06-24
10:10:21,695::vdsmapi::143::SchemaCache::(_report_inconsistency)
Parameter apparentsize is not uint type
jsonrpc.Executor/5::DEBUG::2016-06-24
10:10:21,695::__init__::550::jsonrpc.JsonRpcServer::(_serveRequest)
Return 'Volume.getInfo' in bridge with {'sta
tus': 'OK', 'domain': '46f55a31-f35f-465c-b3e2-df45c05e06a7',
'voltype': 'LEAF', 'description': 'hosted-engine.lockspace',
'parent': '00000000-0000-00
00-0000-000000000000', 'format': 'RAW', 'image':
'6838c974-7656-4b40-87cc-f562ff0b2a4c', 'ctime': '1423074433',
'disktype': '2', 'legality': 'LEGAL',
'mtime': '0', 'apparentsize': '1048576', 'children': [], 'pool': '',
'capacity': '1048576', 'uuid':
u'c66a14d3-112a-4104-9025-76bb2e7ad9f1', 'truesize
': '1048576', 'type': 'PREALLOCATED'}
JsonRpc (StompReactor)::ERROR::2016-06-24
10:10:36,514::betterAsyncore::113::vds.dispatcher::(recv) SSL error
during reading data: (104, 'Connection r
eset by peer')
JsonRpc (StompReactor)::WARNING::2016-06-24
10:10:36,515::betterAsyncore::154::vds.dispatcher::(log_info)
unhandled close event
JsonRpc (StompReactor)::ERROR::2016-06-24
10:10:43,807::betterAsyncore::132::vds.dispatcher::(send) SSL error
during sending data: (104, 'Connection r
eset by peer')
JsonRpc (StompReactor)::ERROR::2016-06-24
10:10:43,959::betterAsyncore::113::vds.dispatcher::(recv) SSL error
during reading data: bad write retry
JsonRpc (StompReactor)::WARNING::2016-06-24
10:10:43,959::betterAsyncore::154::vds.dispatcher::(log_info)
unhandled close event
JsonRpc (StompReactor)::ERROR::2016-06-24
10:10:47,859::betterAsyncore::113::vds.dispatcher::(recv) SSL error
during reading data: (104, 'Connection r
eset by peer')
JsonRpc (StompReactor)::WARNING::2016-06-24
10:10:47,860::betterAsyncore::154::vds.dispatcher::(log_info)
unhandled close event
JsonRpc (StompReactor)::ERROR::2016-06-24
10:10:51,725::betterAsyncore::113::vds.dispatcher::(recv) SSL error
during reading data: (104, 'Connection r
eset by peer')
JsonRpc (StompReactor)::WARNING::2016-06-24
10:10:51,726::betterAsyncore::154::vds.dispatcher::(log_info)
unhandled close event
Reactor thread::INFO::2016-06-24 
10:10:53,851::protocoldetector::72::ProtocolDetector.AcceptorImpl::(handle_accept)
Accepting connection from ::ffff:1
92.168.1.50:48554
Reactor thread::DEBUG::2016-06-24
10:10:53,860::protocoldetector::85::ProtocolDetector.Detector::(__init__)
Using required_size=11
Reactor thread::INFO::2016-06-24 
10:10:53,861::protocoldetector::121::ProtocolDetector.Detector::(handle_read)
Detected protocol stomp from ::ffff:192
.168.1.50:48554
Reactor thread::INFO::2016-06-24
10:10:53,862::stompreactor::101::Broker.StompAdapter::(_cmd_connect)
Processing CONNECT request
Reactor thread::DEBUG::2016-06-24 
10:10:53,862::stompreactor::482::protocoldetector.StompDetector::(handle_socket)
Stomp detected from ('::ffff:192.168.1.50', 48554)



Il 24/06/2016 8.18, Sandro Bonazzola ha scritto:

On Thu, Jun 23, 2016 at 11:46 PM, Stefano Danzi
<s.da...@hawai.it> wrote:

     Hi!
     After cleanin metadata yum do an update of vdsm:

     [root@ovirt01 ~]# rpm -qva | grep vdsm
     vdsm-yajsonrpc-4.18.4.1-0.el7.centos.noarch
     vdsm-infra-4.18.4.1-0.el7.centos.noarch
     vdsm-cli-4.18.4.1-0.el7.centos.noarch
     vdsm-python-4.18.4.1-0.el7.centos.noarch
     vdsm-hook-vmfex-dev-4.18.4.1-0.el7.centos.noarch
     vdsm-xmlrpc-4.18.4.1-0.el7.centos.noarch
     vdsm-4.18.4.1-0.el7.centos.x86_64
     vdsm-api-4.18.4.1-0.el7.centos.noarch
     vdsm-gluster-4.18.4.1-0.el7.centos.noarch
     vdsm-jsonrpc-4.18.4.1-0.el7.centos.noarch

     But this not solve the issue.

     - Host haven't default gateway after a reboot
     - Self Hosted engine don't start.


Martin, Dan, can you please look into this?
Stefano, can you please share a full sos report from the host?


     vdsm.log:
     
https://drive.google.com/file/d/0ByMG4sDqvlZcVEJ5YVI1UWxrdE0/view?usp=sharing



     Il 2016-06-23 21:41 Sandro Bonazzola ha scritto:

         On Thu, Jun 23, 2016 at 6:36 PM, Stefano Danzi
         <s.da...@hawai.it>
         wrote:

             Hi!
             I've just upgrade oVirt from 3.6 to 4.0 and I'm not able
             to start
             the self hosted engine.


         Hi Stefano, can you please try "yum clean metadata" "yum update"
         again?
         You should get vdsm 4.18.4.1, please let us know if this
         solve your
         issue.




--
Sandro Bonazzola
Better technology. Faster innovation. Powered by community
collaboration.
See how it works at redhat.com <http://redhat.com>
Webwww.hawai.it


_______________________________________________
Users mailing list
Users@ovirt.org
http://lists.ovirt.org/mailman/listinfo/users

_______________________________________________
Users mailing list
Users@ovirt.org
http://lists.ovirt.org/mailman/listinfo/users

Reply via email to