Hi
I've noticed the following bug/regression with Tentacle release v20.2.2
Ceph health checks are failing because there are existing raid mirror
volumes on the node hosts, and ceph-volume is trying to read the "raid
internal" logical volumes
[WRN] CEPHADM_REFRESH_FAILED: failed to probe daemons or devices
host zephir `cephadm ceph-volume` failed: cephadm exited with an error
code: 1, stderr: Inferring config
/var/lib/ceph/27923302-87a5-11ec-ac5b-976d21a49941/mon.zephir/config
Non-zero exit code 1 from /usr/bin/podman run --rm --ipc=host
--stop-signal=SIGTERM --net=host --entrypoint /usr/sbin/ceph-volume
--privileged --group-add=disk --init -e CONTAINER_IMAGE=quay.io/cep
h/ceph:v20.2.2 -e NODE_NAME=zephir -e CEPH_VOLUME_SKIP_RESTORECON=yes -e
CEPH_VOLUME_DEBUG=1 -v
/var/run/ceph/27923302-87a5-11ec-ac5b-976d21a49941:/var/run/ceph:z -v
/var/log/ceph/27923302-87a5-11e
c-ac5b-976d21a49941:/var/log/ceph:z -v
/var/lib/ceph/27923302-87a5-11ec-ac5b-976d21a49941/crash:/var/lib/ceph/crash:z
-v /run/systemd/journal:/run/systemd/journal -v /dev:/dev -v
/run/udev:/run/ude
v -v /sys:/sys -v /run/lvm:/run/lvm -v /run/lock/lvm:/run/lock/lvm -v
/:/rootfs:rslave -v /etc/hosts:/etc/hosts:ro -v
/tmp/ceph-tmpe1ewv2jz:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v20.2.2
inventory
--format=json-pretty --with-lsm --filter-for-batch
/usr/bin/podman: stderr stderr: blkid: error: /dev/debian/var_rmeta_0: No
such file or directory
/usr/bin/podman: stderr stderr: blkid: error: /dev/debian/var_rmeta_0: No
such file or directory
/usr/bin/podman: stderr stderr: blkid: error: /dev/debian/var_rmeta_0: No
such file or directory
/usr/bin/podman: stderr stderr: blkid: error: /dev/debian/var_rmeta_0: No
such file or directory
/usr/bin/podman: stderr stderr: blkid: error: /dev/debian/var_rmeta_0: No
such file or directory
/usr/bin/podman: stderr stderr: blkid: error: /dev/debian/var_rmeta_0: No
such file or directory
/usr/bin/podman: stderr stderr: blkid: error: /dev/debian/var_rmeta_0: No
such file or directory
/usr/bin/podman: stderr stderr: blkid: error: /dev/debian/var_rmeta_0: No
such file or directory
/usr/bin/podman: stderr stderr: blkid: error: /dev/debian/var_rmeta_0: No
such file or directory
/usr/bin/podman: stderr stderr: blkid: error: /dev/debian/var_rmeta_0: No
such file or directory
/usr/bin/podman: stderr stderr: blkid: error: /dev/debian/var_rmeta_0: No
such file or directory
/usr/bin/podman: stderr stderr: blkid: error: /dev/debian/var_rmeta_0: No
such file or directory
/usr/bin/podman: stderr Traceback (most recent call last):
/usr/bin/podman: stderr File "/usr/sbin/ceph-volume", line 33, in <module>
/usr/bin/podman: stderr sys.exit(load_entry_point('ceph-volume==1.0.0',
'console_scripts', 'ceph-volume')())
/usr/bin/podman: stderr File
"/usr/lib/python3.9/site-packages/ceph_volume/main.py", line 62, in __init__
/usr/bin/podman: stderr self.main(self.argv)
/usr/bin/podman: stderr File
"/usr/lib/python3.9/site-packages/ceph_volume/decorators.py", line 59, in
newfunc
/usr/bin/podman: stderr return f(*a, **kw)
/usr/bin/podman: stderr File
"/usr/lib/python3.9/site-packages/ceph_volume/main.py", line 174, in main
/usr/bin/podman: stderr terminal.dispatch(self.mapper, subcommand_args)
/usr/bin/podman: stderr File
"/usr/lib/python3.9/site-packages/ceph_volume/terminal.py", line 194, in
dispatch
/usr/bin/podman: stderr instance.main()
/usr/bin/podman: stderr File
"/usr/lib/python3.9/site-packages/ceph_volume/inventory/main.py", line 58,
in main
/usr/bin/podman: stderr
self.format_report(Devices(filter_for_batch=self.args.filter_for_batch,
/usr/bin/podman: stderr File
"/usr/lib/python3.9/site-packages/ceph_volume/util/device.py", line 43, in
__init__
/usr/bin/podman: stderr self._devices = [Device(k,
/usr/bin/podman: stderr File
"/usr/lib/python3.9/site-packages/ceph_volume/util/device.py", line 43, in
<listcomp>
/usr/bin/podman: stderr self._devices = [Device(k,
/usr/bin/podman: stderr File
"/usr/lib/python3.9/site-packages/ceph_volume/util/device.py", line 156, in
__init__
/usr/bin/podman: stderr self.device_id = self._get_device_id()
/usr/bin/podman: stderr File
"/usr/lib/python3.9/site-packages/ceph_volume/util/device.py", line 322, in
_get_device_id
/usr/bin/podman: stderr udev_data = disk.UdevData(self.path)
/usr/bin/podman: stderr File
"/usr/lib/python3.9/site-packages/ceph_volume/util/disk.py", line 1355, in
__init__
/usr/bin/podman: stderr raise RuntimeError(f'{path} not found.')
/usr/bin/podman: stderr RuntimeError: /dev/debian/var_rmeta_0 not found.
Traceback (most recent call last):
File "<frozen runpy>", line 198, in _run_module_as_main
File "<frozen runpy>", line 88, in _run_code
File
"/var/lib/ceph/27923302-87a5-11ec-ac5b-976d21a49941/cephadm.42daa0d45411be4c8bb16fe92e265c59cc21fc86cd0040b96409c80ba0da884c/__main__.py",
line 5296, in <module>
main()
~~~~^^
File
"/var/lib/ceph/27923302-87a5-11ec-ac5b-976d21a49941/cephadm.42daa0d45411be4c8bb16fe92e265c59cc21fc86cd0040b96409c80ba0da884c/__main__.py",
line 5284, in main
r = ctx.func(ctx)
File
"/var/lib/ceph/27923302-87a5-11ec-ac5b-976d21a49941/cephadm.42daa0d45411be4c8bb16fe92e265c59cc21fc86cd0040b96409c80ba0da884c/__main__.py",
line 397, in _infer_config
return func(ctx)
File
"/var/lib/ceph/27923302-87a5-11ec-ac5b-976d21a49941/cephadm.42daa0d45411be4c8bb16fe92e265c59cc21fc86cd0040b96409c80ba0da884c/__main__.py",
line 312, in _infer_fsid
return func(ctx)
File
"/var/lib/ceph/27923302-87a5-11ec-ac5b-976d21a49941/cephadm.42daa0d45411be4c8bb16fe92e265c59cc21fc86cd0040b96409c80ba0da884c/__main__.py",
line 425, in _infer_image
return func(ctx)
File
"/var/lib/ceph/27923302-87a5-11ec-ac5b-976d21a49941/cephadm.42daa0d45411be4c8bb16fe92e265c59cc21fc86cd0040b96409c80ba0da884c/__main__.py",
line 299, in _validate_fsid
return func(ctx)
File
"/var/lib/ceph/27923302-87a5-11ec-ac5b-976d21a49941/cephadm.42daa0d45411be4c8bb16fe92e265c59cc21fc86cd0040b96409c80ba0da884c/__main__.py",
line 3231, in command_ceph_volume
out, err, code = call_throws(ctx, c.run_cmd(),
verbosity=CallVerbosity.QUIET_UNLESS_ERROR)
~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/var/lib/ceph/27923302-87a5-11ec-ac5b-976d21a49941/cephadm.42daa0d45411be4c8bb16fe92e265c59cc21fc86cd0040b96409c80ba0da884c/cephadmlib/call_wrappers.py",
line 310, in call_throws
raise RuntimeError('Failed command: %s' % ' '.join(command))
RuntimeError: Failed command: /usr/bin/podman run --rm --ipc=host
--stop-signal=SIGTERM --net=host --entrypoint /usr/sbin/ceph-volume
--privileged --group-add=disk --init -e CONTAINER_IMAGE=quay.io
/ceph/ceph:v20.2.2 -e NODE_NAME=zephir -e CEPH_VOLUME_SKIP_RESTORECON=yes
-e CEPH_VOLUME_DEBUG=1 -v
/var/run/ceph/27923302-87a5-11ec-ac5b-976d21a49941:/var/run/ceph:z -v
/var/log/ceph/27923302-87a5
-11ec-ac5b-976d21a49941:/var/log/ceph:z -v
/var/lib/ceph/27923302-87a5-11ec-ac5b-976d21a49941/crash:/var/lib/ceph/crash:z
-v /run/systemd/journal:/run/systemd/journal -v /dev:/dev -v /run/udev:/run
/udev -v /sys:/sys -v /run/lvm:/run/lvm -v /run/lock/lvm:/run/lock/lvm -v
/:/rootfs:rslave -v /etc/hosts:/etc/hosts:ro -v
/tmp/ceph-tmpe1ewv2jz:/etc/ceph/ceph.conf:z quay.io/ceph/ceph:v20.2.2 inven
tory --format=json-pretty --with-lsm --filter-for-batch
lvs output example:
root@zephir:~# lvs -a -o +role -o +devices |head -1; lvs -a -o +role -o
+devices| grep var
LV VG
Attr LSize Pool Origin
Data% Meta% Move Log Cpy%Sync Convert Role
Devices
var debian
rwi-aor--- <139.70g
100.00
public
var_rimage_0(0),var_rimage_1(0)
var_backup debian
-wi-ao---- 120.00g
public
/dev/mapper/sdc1_crypt(23040)
[var_rimage_0] debian
iwi-aor--- <139.70g
private,ra
id,image /dev/mapper/nvme0n1p3_crypt(71525)
[var_rimage_1] debian
iwi-aor--- <139.70g
private,ra
id,image /dev/mapper/nvme1n1p1_crypt(214578)
[var_rmeta_0] debian
ewi-aor--- 4.00m
private,ra
id,metadata /dev/mapper/nvme0n1p3_crypt(0)
[var_rmeta_1] debian
ewi-aor--- 4.00m
private,ra
id,metadata /dev/mapper/nvme1n1p1_crypt(214577)
var-lib-ceph debian-4k
rwi-aor--- 40.00g
100.00
public
var-lib-ceph_rimage_0(0),var-lib-ceph_rimage_1(0)
[var-lib-ceph_rimage_0] debian-4k
iwi-aor--- 40.00g
private,ra
id,image /dev/mapper/nvme3n1_crypt(55050)
[var-lib-ceph_rimage_1] debian-4k
iwi-aor--- 40.00g
private,ra
id,image /dev/mapper/nvme4n1_crypt(786433)
[var-lib-ceph_rmeta_0] debian-4k
ewi-aor--- 4.00m
private,ra
id,metadata /dev/mapper/nvme3n1_crypt(55049)
[var-lib-ceph_rmeta_1] debian-4k
ewi-aor--- 4.00m
private,ra
id,metadata /dev/mapper/nvme4n1_crypt(786432)
root@zephir:~#
Maybe ceph-volume should only try to read the logical volumes with Role
'public' and not the lv raid internal ones.
Those volumes are used to host the standard OS installation directories
(/,/var,etc) on mirrored volumes.
>
> ceph-volume
> -----------
>
> * Inventory Scanning: Enhanced inventory discovery logic to
> automatically skip RAM disk devices (/dev/ram*).
>
> To unsubscribe send an email to [email protected]
Thanks & Regards
Reto
_______________________________________________
ceph-users mailing list -- [email protected]
To unsubscribe send an email to [email protected]
