From bd7e78cf479cd698195c5cef6a97b6637d79e34e Mon Sep 17 00:00:00 2001 From: Nic Anderson Date: Thu, 1 Feb 2024 23:44:22 -0500 Subject: [PATCH] update to v18.2.1 with staged reef backport --- Dockerfile | 2 +- src/ceph-volume/ceph_volume/util/device.py | 55 ++++++--- src/ceph-volume/ceph_volume/util/disk.py | 127 ++++++++++----------- 3 files changed, 97 insertions(+), 87 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3d3d6b5..78bb9a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/ceph/ceph:v17.2.5 +FROM quay.io/ceph/ceph:v18.2.1 LABEL org.opencontainers.image.source https://github.com/nanderson94/ceph-patch diff --git a/src/ceph-volume/ceph_volume/util/device.py b/src/ceph-volume/ceph_volume/util/device.py index e5ebb7d..932903d 100644 --- a/src/ceph-volume/ceph_volume/util/device.py +++ b/src/ceph-volume/ceph_volume/util/device.py @@ -33,20 +33,30 @@ class Devices(object): A container for Device instances with reporting """ - def __init__(self, filter_for_batch=False, with_lsm=False): + def __init__(self, + filter_for_batch=False, + with_lsm=False, + list_all=False): lvs = lvm.get_lvs() lsblk_all = disk.lsblk_all() all_devices_vgs = lvm.get_all_devices_vgs() if not sys_info.devices: sys_info.devices = disk.get_devices() - self.devices = [Device(k, - with_lsm, - lvs=lvs, - lsblk_all=lsblk_all, - all_devices_vgs=all_devices_vgs) for k in - sys_info.devices.keys()] - if filter_for_batch: - self.devices = [d for d in self.devices if d.available_lvm_batch] + self._devices = [Device(k, + with_lsm, + lvs=lvs, + lsblk_all=lsblk_all, + all_devices_vgs=all_devices_vgs) for k in + sys_info.devices.keys()] + self.devices = [] + for device in self._devices: + if filter_for_batch and not device.available_lvm_batch: + continue + if device.is_lv and not list_all: + continue + if device.is_partition and not list_all: + continue + self.devices.append(device) def pretty_report(self): output = [ @@ -111,13 +121,8 @@ class Device(object): # check if we are not a device mapper if "dm-" not in real_path: self.path = real_path - if not sys_info.devices: - if self.path: - sys_info.devices = disk.get_devices(device=self.path) - else: - sys_info.devices = disk.get_devices() - if sys_info.devices.get(self.path, {}): - self.device_nodes = sys_info.devices[self.path]['device_nodes'] + if not sys_info.devices.get(self.path): + sys_info.devices = disk.get_devices() self.sys_api = sys_info.devices.get(self.path, {}) self.partitions = self._get_partitions() self.lv_api = None @@ -133,6 +138,8 @@ class Device(object): self._is_lvm_member = None self.ceph_device = False self._parse() + if self.path in sys_info.devices.keys(): + self.device_nodes = sys_info.devices[self.path]['device_nodes'] self.lsm_data = self.fetch_lsm(with_lsm) self.available_lvm, self.rejected_reasons_lvm = self._check_lvm_reject_reasons() @@ -230,7 +237,7 @@ class Device(object): self.disk_api = dev device_type = dev.get('TYPE', '') # always check is this is an lvm member - valid_types = ['part', 'disk'] + valid_types = ['part', 'disk', 'mpath'] if allow_loop_devices(): valid_types.append('loop') if device_type in valid_types: @@ -490,7 +497,7 @@ class Device(object): @property def is_acceptable_device(self): - return self.is_device or self.is_partition + return self.is_device or self.is_partition or self.is_lv @property def is_encrypted(self): @@ -528,6 +535,15 @@ class Device(object): if lv.tags.get("ceph.type") in ["data", "block"]] return any(osd_ids) + @property + def journal_used_by_ceph(self): + # similar to used_by_ceph() above. This is for 'journal' devices (db/wal/..) + # needed by get_lvm_fast_allocs() in devices/lvm/batch.py + # see https://tracker.ceph.com/issues/59640 + osd_ids = [lv.tags.get("ceph.osd_id") is not None for lv in self.lvs + if lv.tags.get("ceph.type") in ["db", "wal"]] + return any(osd_ids) + @property def vg_free_percent(self): if self.vgs: @@ -577,7 +593,6 @@ class Device(object): reasons = [ ('id_bus', 'usb', 'id_bus'), ('ro', '1', 'read-only'), - ('locked', 1, 'locked'), ] rejected = [reason for (k, v, reason) in reasons if self.sys_api.get(k, '') == v] @@ -613,6 +628,8 @@ class Device(object): rejected.append('Has GPT headers') if self.has_partitions: rejected.append('Has partitions') + if self.has_fs: + rejected.append('Has a FileSystem') return rejected def _check_lvm_reject_reasons(self): diff --git a/src/ceph-volume/ceph_volume/util/disk.py b/src/ceph-volume/ceph_volume/util/disk.py index c55645a..01c3899 100644 --- a/src/ceph-volume/ceph_volume/util/disk.py +++ b/src/ceph-volume/ceph_volume/util/disk.py @@ -6,6 +6,7 @@ import time from ceph_volume import process from ceph_volume.api import lvm from ceph_volume.util.system import get_file_contents +from typing import Dict, List logger = logging.getLogger(__name__) @@ -238,11 +239,14 @@ def _udevadm_info(device): def lsblk(device, columns=None, abspath=False): - result = lsblk_all(device=device, - columns=columns, - abspath=abspath) + result = [] + if not os.path.isdir(device): + result = lsblk_all(device=device, + columns=columns, + abspath=abspath) if not result: - raise RuntimeError(f"{device} not found is lsblk report") + logger.debug(f"{device} not found is lsblk report") + return {} return result[0] @@ -356,31 +360,23 @@ def is_device(dev): if not allow_loop_devices(): return False + TYPE = lsblk(dev).get('TYPE') + if TYPE: + return TYPE in ['disk', 'mpath'] + # fallback to stat - return _stat_is_device(os.lstat(dev).st_mode) + return _stat_is_device(os.lstat(dev).st_mode) and not is_partition(dev) -def is_partition(dev): +def is_partition(dev: str) -> bool: """ Boolean to determine if a given device is a partition, like /dev/sda1 """ if not os.path.exists(dev): return False - # use lsblk first, fall back to using stat - TYPE = lsblk(dev).get('TYPE') - if TYPE: - return TYPE == 'part' - # fallback to stat - stat_obj = os.stat(dev) - if _stat_is_device(stat_obj.st_mode): - return False - - major = os.major(stat_obj.st_rdev) - minor = os.minor(stat_obj.st_rdev) - if os.path.exists('/sys/dev/block/%d:%d/partition' % (major, minor)): - return True - return False + partitions = get_partitions() + return dev.split("/")[-1] in partitions def is_ceph_rbd(dev): @@ -731,28 +727,6 @@ def is_mapper_device(device_name): return device_name.startswith(('/dev/mapper', '/dev/dm-')) -def is_locked_raw_device(disk_path): - """ - A device can be locked by a third party software like a database. - To detect that case, the device is opened in Read/Write and exclusive mode - """ - open_flags = (os.O_RDWR | os.O_EXCL) - open_mode = 0 - fd = None - - try: - fd = os.open(disk_path, open_flags, open_mode) - except OSError: - return 1 - - try: - os.close(fd) - except OSError: - return 1 - - return 0 - - class AllowLoopDevices(object): allow = False warned = False @@ -778,34 +752,34 @@ class AllowLoopDevices(object): allow_loop_devices = AllowLoopDevices() -def get_block_devs_sysfs(_sys_block_path='/sys/block', _sys_dev_block_path='/sys/dev/block', device=''): - def holder_inner_loop(): +def get_block_devs_sysfs(_sys_block_path: str = '/sys/block', _sys_dev_block_path: str = '/sys/dev/block', device: str = '') -> List[List[str]]: + def holder_inner_loop() -> bool: for holder in holders: # /sys/block/sdy/holders/dm-8/dm/uuid - holder_dm_type = get_file_contents(os.path.join(_sys_block_path, dev, f'holders/{holder}/dm/uuid')).split('-')[0].lower() + holder_dm_type: str = get_file_contents(os.path.join(_sys_block_path, dev, f'holders/{holder}/dm/uuid')).split('-')[0].lower() if holder_dm_type == 'mpath': return True # First, get devices that are _not_ partitions - result = list() + result: List[List[str]] = list() if not device: - dev_names = os.listdir(_sys_block_path) + dev_names: List[str] = os.listdir(_sys_block_path) else: dev_names = [device] for dev in dev_names: - name = kname = os.path.join("/dev", dev) + name = kname = pname = os.path.join("/dev", dev) if not os.path.exists(name): continue - type_ = 'disk' - holders = os.listdir(os.path.join(_sys_block_path, dev, 'holders')) + type_: str = 'disk' + holders: List[str] = os.listdir(os.path.join(_sys_block_path, dev, 'holders')) if holder_inner_loop(): continue - dm_dir_path = os.path.join(_sys_block_path, dev, 'dm') + dm_dir_path: str = os.path.join(_sys_block_path, dev, 'dm') if os.path.isdir(dm_dir_path): - dm_type = get_file_contents(os.path.join(dm_dir_path, 'uuid')) - type_ = dm_type.split('-')[0].lower() - basename = get_file_contents(os.path.join(dm_dir_path, 'name')) - name = os.path.join("/dev/mapper", basename) + dm_type: str = get_file_contents(os.path.join(dm_dir_path, 'uuid')) + type_: List[str] = dm_type.split('-')[0].lower() + basename: str = get_file_contents(os.path.join(dm_dir_path, 'name')) + name: str = os.path.join("/dev/mapper", basename) if dev.startswith('loop'): if not allow_loop_devices(): continue @@ -813,17 +787,27 @@ def get_block_devs_sysfs(_sys_block_path='/sys/block', _sys_dev_block_path='/sys if not os.path.exists(os.path.join(_sys_block_path, dev, 'loop')): continue type_ = 'loop' - result.append([kname, name, type_]) + result.append([kname, name, type_, pname]) # Next, look for devices that _are_ partitions - for item in os.listdir(_sys_dev_block_path): - is_part = get_file_contents(os.path.join(_sys_dev_block_path, item, 'partition')) == "1" - dev = os.path.basename(os.readlink(os.path.join(_sys_dev_block_path, item))) - if not is_part: - continue - name = kname = os.path.join("/dev", dev) - result.append([name, kname, "part"]) + partitions: Dict[str, str] = get_partitions() + for partition in partitions.keys(): + name = kname = os.path.join("/dev", partition) + result.append([name, kname, "part", partitions[partition]]) return sorted(result, key=lambda x: x[0]) +def get_partitions(_sys_dev_block_path ='/sys/dev/block') -> List[str]: + devices: List[str] = os.listdir(_sys_dev_block_path) + result: Dict[str, str] = dict() + for device in devices: + device_path: str = os.path.join(_sys_dev_block_path, device) + is_partition: bool = int(get_file_contents(os.path.join(device_path, 'partition'), '0')) > 0 + if not is_partition: + continue + + partition_sys_name: str = os.path.basename(os.path.realpath(device_path)) + parent_device_sys_name: str = os.path.realpath(device_path).split('/')[-2:-1][0] + result[partition_sys_name] = parent_device_sys_name + return result def get_devices(_sys_block_path='/sys/block', device=''): """ @@ -840,16 +824,20 @@ def get_devices(_sys_block_path='/sys/block', device=''): block_devs = get_block_devs_sysfs(_sys_block_path) - block_types = ['disk', 'mpath'] + block_types = ['disk', 'mpath', 'lvm', 'part'] if allow_loop_devices(): block_types.append('loop') for block in block_devs: + if block[2] == 'lvm': + block[1] = lvm.get_lv_path_from_mapper(block[1]) devname = os.path.basename(block[0]) diskname = block[1] if block[2] not in block_types: continue sysdir = os.path.join(_sys_block_path, devname) + if block[2] == 'part': + sysdir = os.path.join(_sys_block_path, block[3], devname) metadata = {} # If the device is ceph rbd it gets excluded @@ -877,11 +865,17 @@ def get_devices(_sys_block_path='/sys/block', device=''): for key, file_ in facts: metadata[key] = get_file_contents(os.path.join(sysdir, file_)) - device_slaves = os.listdir(os.path.join(sysdir, 'slaves')) + if block[2] != 'part': + device_slaves = os.listdir(os.path.join(sysdir, 'slaves')) + metadata['partitions'] = get_partitions_facts(sysdir) + if device_slaves: metadata['device_nodes'] = ','.join(device_slaves) else: - metadata['device_nodes'] = devname + if block[2] == 'part': + metadata['device_nodes'] = block[3] + else: + metadata['device_nodes'] = devname metadata['actuators'] = None if os.path.isdir(sysdir + "/queue/independent_access_ranges/"): @@ -909,7 +903,6 @@ def get_devices(_sys_block_path='/sys/block', device=''): metadata['size'] = float(size) * 512 metadata['human_readable_size'] = human_readable_size(metadata['size']) metadata['path'] = diskname - metadata['locked'] = is_locked_raw_device(metadata['path']) metadata['type'] = block[2] # some facts from udevadm