update to v18.2.1 with staged reef backport

Dockerfile

@@ -1,4 +1,4 @@
-FROM quay.io/ceph/ceph:v17.2.5
+FROM quay.io/ceph/ceph:v18.2.1
 
 LABEL org.opencontainers.image.source https://github.com/nanderson94/ceph-patch
 

src/ceph-volume/ceph_volume/util/device.py

@@ -33,20 +33,30 @@ class Devices(object):
     A container for Device instances with reporting
     """
 
-    def __init__(self, filter_for_batch=False, with_lsm=False):
+    def __init__(self,
+                 filter_for_batch=False,
+                 with_lsm=False,
+                 list_all=False):
         lvs = lvm.get_lvs()
         lsblk_all = disk.lsblk_all()
         all_devices_vgs = lvm.get_all_devices_vgs()
         if not sys_info.devices:
             sys_info.devices = disk.get_devices()
-        self.devices = [Device(k,
+        self._devices = [Device(k,
                                 with_lsm,
                                 lvs=lvs,
                                 lsblk_all=lsblk_all,
                                 all_devices_vgs=all_devices_vgs) for k in
                          sys_info.devices.keys()]
-        if filter_for_batch:
+        self.devices = []
-            self.devices = [d for d in self.devices if d.available_lvm_batch]
+        for device in self._devices:
+            if filter_for_batch and not device.available_lvm_batch:
+                continue
+            if device.is_lv and not list_all:
+                continue
+            if device.is_partition and not list_all:
+                continue
+            self.devices.append(device)
 
     def pretty_report(self):
         output = [
@@ -111,13 +121,8 @@ class Device(object):
             # check if we are not a device mapper
             if "dm-" not in real_path:
                 self.path = real_path
-        if not sys_info.devices:
+        if not sys_info.devices.get(self.path):
-            if self.path:
-                sys_info.devices = disk.get_devices(device=self.path)
-            else:
             sys_info.devices = disk.get_devices()
-        if sys_info.devices.get(self.path, {}):
-            self.device_nodes = sys_info.devices[self.path]['device_nodes']
         self.sys_api = sys_info.devices.get(self.path, {})
         self.partitions = self._get_partitions()
         self.lv_api = None
@@ -133,6 +138,8 @@ class Device(object):
         self._is_lvm_member = None
         self.ceph_device = False
         self._parse()
+        if self.path in sys_info.devices.keys():
+            self.device_nodes = sys_info.devices[self.path]['device_nodes']
         self.lsm_data = self.fetch_lsm(with_lsm)
 
         self.available_lvm, self.rejected_reasons_lvm = self._check_lvm_reject_reasons()
@@ -230,7 +237,7 @@ class Device(object):
             self.disk_api = dev
             device_type = dev.get('TYPE', '')
             # always check is this is an lvm member
-            valid_types = ['part', 'disk']
+            valid_types = ['part', 'disk', 'mpath']
             if allow_loop_devices():
                 valid_types.append('loop')
             if device_type in valid_types:
@@ -490,7 +497,7 @@ class Device(object):
 
     @property
     def is_acceptable_device(self):
-        return self.is_device or self.is_partition
+        return self.is_device or self.is_partition or self.is_lv
 
     @property
     def is_encrypted(self):
@@ -528,6 +535,15 @@ class Device(object):
                    if lv.tags.get("ceph.type") in ["data", "block"]]
         return any(osd_ids)
 
+    @property
+    def journal_used_by_ceph(self):
+        # similar to used_by_ceph() above. This is for 'journal' devices (db/wal/..)
+        # needed by get_lvm_fast_allocs() in devices/lvm/batch.py
+        # see https://tracker.ceph.com/issues/59640
+        osd_ids = [lv.tags.get("ceph.osd_id") is not None for lv in self.lvs
+                   if lv.tags.get("ceph.type") in ["db", "wal"]]
+        return any(osd_ids)
+
     @property
     def vg_free_percent(self):
         if self.vgs:
@@ -577,7 +593,6 @@ class Device(object):
         reasons = [
             ('id_bus', 'usb', 'id_bus'),
             ('ro', '1', 'read-only'),
-            ('locked', 1, 'locked'),
         ]
         rejected = [reason for (k, v, reason) in reasons if
                     self.sys_api.get(k, '') == v]
@@ -613,6 +628,8 @@ class Device(object):
             rejected.append('Has GPT headers')
         if self.has_partitions:
             rejected.append('Has partitions')
+        if self.has_fs:
+            rejected.append('Has a FileSystem')
         return rejected
 
     def _check_lvm_reject_reasons(self):

src/ceph-volume/ceph_volume/util/disk.py

@@ -6,6 +6,7 @@ import time
 from ceph_volume import process
 from ceph_volume.api import lvm
 from ceph_volume.util.system import get_file_contents
+from typing import Dict, List
 
 
 logger = logging.getLogger(__name__)
@@ -238,11 +239,14 @@ def _udevadm_info(device):
 
 
 def lsblk(device, columns=None, abspath=False):
+    result = []
+    if not os.path.isdir(device):
         result = lsblk_all(device=device,
                            columns=columns,
                            abspath=abspath)
     if not result:
-        raise RuntimeError(f"{device} not found is lsblk report")
+        logger.debug(f"{device} not found is lsblk report")
+        return {}
 
     return result[0]
 
@@ -356,31 +360,23 @@ def is_device(dev):
         if not allow_loop_devices():
             return False
 
+    TYPE = lsblk(dev).get('TYPE')
+    if TYPE:
+        return TYPE in ['disk', 'mpath']
+
     # fallback to stat
-    return _stat_is_device(os.lstat(dev).st_mode)
+    return _stat_is_device(os.lstat(dev).st_mode) and not is_partition(dev)
 
 
-def is_partition(dev):
+def is_partition(dev: str) -> bool:
     """
     Boolean to determine if a given device is a partition, like /dev/sda1
     """
     if not os.path.exists(dev):
         return False
-    # use lsblk first, fall back to using stat
-    TYPE = lsblk(dev).get('TYPE')
-    if TYPE:
-        return TYPE == 'part'
 
-    # fallback to stat
+    partitions = get_partitions()
-    stat_obj = os.stat(dev)
+    return dev.split("/")[-1] in partitions
-    if _stat_is_device(stat_obj.st_mode):
-        return False
-
-    major = os.major(stat_obj.st_rdev)
-    minor = os.minor(stat_obj.st_rdev)
-    if os.path.exists('/sys/dev/block/%d:%d/partition' % (major, minor)):
-        return True
-    return False
 
 
 def is_ceph_rbd(dev):
@@ -731,28 +727,6 @@ def is_mapper_device(device_name):
     return device_name.startswith(('/dev/mapper', '/dev/dm-'))
 
 
-def is_locked_raw_device(disk_path):
-    """
-    A device can be locked by a third party software like a database.
-    To detect that case, the device is opened in Read/Write and exclusive mode
-    """
-    open_flags = (os.O_RDWR | os.O_EXCL)
-    open_mode = 0
-    fd = None
-
-    try:
-        fd = os.open(disk_path, open_flags, open_mode)
-    except OSError:
-        return 1
-
-    try:
-        os.close(fd)
-    except OSError:
-        return 1
-
-    return 0
-
-
 class AllowLoopDevices(object):
     allow = False
     warned = False
@@ -778,34 +752,34 @@ class AllowLoopDevices(object):
 allow_loop_devices = AllowLoopDevices()
 
 
-def get_block_devs_sysfs(_sys_block_path='/sys/block', _sys_dev_block_path='/sys/dev/block', device=''):
+def get_block_devs_sysfs(_sys_block_path: str = '/sys/block', _sys_dev_block_path: str = '/sys/dev/block', device: str = '') -> List[List[str]]:
-    def holder_inner_loop():
+    def holder_inner_loop() -> bool:
         for holder in holders:
             # /sys/block/sdy/holders/dm-8/dm/uuid
-            holder_dm_type = get_file_contents(os.path.join(_sys_block_path, dev, f'holders/{holder}/dm/uuid')).split('-')[0].lower()
+            holder_dm_type: str = get_file_contents(os.path.join(_sys_block_path, dev, f'holders/{holder}/dm/uuid')).split('-')[0].lower()
             if holder_dm_type == 'mpath':
                 return True
 
     # First, get devices that are _not_ partitions
-    result = list()
+    result: List[List[str]] = list()
     if not device:
-        dev_names = os.listdir(_sys_block_path)
+        dev_names: List[str] = os.listdir(_sys_block_path)
     else:
         dev_names = [device]
     for dev in dev_names:
-        name = kname = os.path.join("/dev", dev)
+        name = kname = pname = os.path.join("/dev", dev)
         if not os.path.exists(name):
             continue
-        type_ = 'disk'
+        type_: str = 'disk'
-        holders = os.listdir(os.path.join(_sys_block_path, dev, 'holders'))
+        holders: List[str] = os.listdir(os.path.join(_sys_block_path, dev, 'holders'))
         if holder_inner_loop():
             continue
-        dm_dir_path = os.path.join(_sys_block_path, dev, 'dm')
+        dm_dir_path: str = os.path.join(_sys_block_path, dev, 'dm')
         if os.path.isdir(dm_dir_path):
-            dm_type = get_file_contents(os.path.join(dm_dir_path, 'uuid'))
+            dm_type: str = get_file_contents(os.path.join(dm_dir_path, 'uuid'))
-            type_ = dm_type.split('-')[0].lower()
+            type_: List[str] = dm_type.split('-')[0].lower()
-            basename = get_file_contents(os.path.join(dm_dir_path, 'name'))
+            basename: str = get_file_contents(os.path.join(dm_dir_path, 'name'))
-            name = os.path.join("/dev/mapper", basename)
+            name: str = os.path.join("/dev/mapper", basename)
         if dev.startswith('loop'):
             if not allow_loop_devices():
                 continue
@@ -813,17 +787,27 @@ def get_block_devs_sysfs(_sys_block_path='/sys/block', _sys_dev_block_path='/sys
             if not os.path.exists(os.path.join(_sys_block_path, dev, 'loop')):
                 continue
             type_ = 'loop'
-        result.append([kname, name, type_])
+        result.append([kname, name, type_, pname])
     # Next, look for devices that _are_ partitions
-    for item in os.listdir(_sys_dev_block_path):
+    partitions: Dict[str, str] = get_partitions()
-        is_part = get_file_contents(os.path.join(_sys_dev_block_path, item, 'partition')) == "1"
+    for partition in partitions.keys():
-        dev = os.path.basename(os.readlink(os.path.join(_sys_dev_block_path, item)))
+        name = kname = os.path.join("/dev", partition)
-        if not is_part:
+        result.append([name, kname, "part", partitions[partition]])
-            continue
-        name = kname = os.path.join("/dev", dev)
-        result.append([name, kname, "part"])
     return sorted(result, key=lambda x: x[0])
 
+def get_partitions(_sys_dev_block_path ='/sys/dev/block') -> List[str]:
+    devices: List[str] = os.listdir(_sys_dev_block_path)
+    result: Dict[str, str] = dict()
+    for device in devices:
+        device_path: str = os.path.join(_sys_dev_block_path, device)
+        is_partition: bool = int(get_file_contents(os.path.join(device_path, 'partition'), '0')) > 0
+        if not is_partition:
+            continue
+
+        partition_sys_name: str = os.path.basename(os.path.realpath(device_path))
+        parent_device_sys_name: str = os.path.realpath(device_path).split('/')[-2:-1][0]
+        result[partition_sys_name] = parent_device_sys_name
+    return result
 
 def get_devices(_sys_block_path='/sys/block', device=''):
     """
@@ -840,16 +824,20 @@ def get_devices(_sys_block_path='/sys/block', device=''):
 
     block_devs = get_block_devs_sysfs(_sys_block_path)
 
-    block_types = ['disk', 'mpath']
+    block_types = ['disk', 'mpath', 'lvm', 'part']
     if allow_loop_devices():
         block_types.append('loop')
 
     for block in block_devs:
+        if block[2] == 'lvm':
+            block[1] = lvm.get_lv_path_from_mapper(block[1])
         devname = os.path.basename(block[0])
         diskname = block[1]
         if block[2] not in block_types:
             continue
         sysdir = os.path.join(_sys_block_path, devname)
+        if block[2] == 'part':
+            sysdir = os.path.join(_sys_block_path, block[3], devname)
         metadata = {}
 
         # If the device is ceph rbd it gets excluded
@@ -877,9 +865,15 @@ def get_devices(_sys_block_path='/sys/block', device=''):
         for key, file_ in facts:
             metadata[key] = get_file_contents(os.path.join(sysdir, file_))
 
+        if block[2] != 'part':
             device_slaves = os.listdir(os.path.join(sysdir, 'slaves'))
+            metadata['partitions'] = get_partitions_facts(sysdir)
+
         if device_slaves:
             metadata['device_nodes'] = ','.join(device_slaves)
+        else:
+            if block[2] == 'part':
+                metadata['device_nodes'] = block[3]
             else:
                 metadata['device_nodes'] = devname
 
@@ -909,7 +903,6 @@ def get_devices(_sys_block_path='/sys/block', device=''):
         metadata['size'] = float(size) * 512
         metadata['human_readable_size'] = human_readable_size(metadata['size'])
         metadata['path'] = diskname
-        metadata['locked'] = is_locked_raw_device(metadata['path'])
         metadata['type'] = block[2]
 
         # some facts from udevadm