From 1779febd5f6740a844b147291c3cfd3e24b76ae2 Mon Sep 17 00:00:00 2001 From: Alexey Sheplyakov Date: Tue, 29 Dec 2015 16:29:06 +0300 Subject: [PATCH] ceph-disk: improve base device/partition number calculation split_dev_base_partnum fails to handle the device nodes names containing digits (like those /dev/disk/by-id generated by udev), as a result ceph-deploy osd prepare $node:/dev/sdc3:/dev/disk/by-id/wwn-0x50014ee00386a548-part3 fails, although it works fine this way ceph-deploy osd prepare $node:/dev/sdc3:/dev/sde3 where /dev/sde3 is the very same partition (i.e. it's the destination of the /dev/disk/by-id/wwn-0x50014ee00386a548-part3 symlink). The problem has been silently fixed in the master branch by commits https://github.com/ceph/ceph/commit/0e34742b968e https://github.com/ceph/ceph/commit/3bc95dfc1b88 https://github.com/ceph/ceph/commit/77ff7c3dc6dd Cherry pick those, and https://github.com/ceph/ceph/commit/1c5fea67 which fixes yet another symlink related problem. Closes-Bug: #1529841 Change-Id: I37f83266efc4468103afd3fe82336b83f91fbe58 --- debian/changelog | 12 ++ ...ed-output-with-the-dereferenced-path.patch | 30 ++++ ...path-predicate-for-multipath-devices.patch | 89 +++++++++++ ...ath-support-for-is_partition-and-lis.patch | 143 ++++++++++++++++++ ...ath-support-for-split_dev_base_partn.patch | 61 ++++++++ debian/patches/series | 4 + 6 files changed, 339 insertions(+) create mode 100644 debian/patches/Compare-parted-output-with-the-dereferenced-path.patch create mode 100644 debian/patches/ceph-disk-is_mpath-predicate-for-multipath-devices.patch create mode 100644 debian/patches/ceph-disk-multipath-support-for-is_partition-and-lis.patch create mode 100644 debian/patches/ceph-disk-multipath-support-for-split_dev_base_partn.patch diff --git a/debian/changelog b/debian/changelog index 20e541fc..08ec5bdb 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +ceph (0.94.5-0u~u14.04+mos2) mos8.0; urgency=low + + * Backport upstream patches which fix the ceph-disk vs udev generated + symlinks problem: + - https://github.com/ceph/ceph/commit/0e34742b968e + - https://github.com/ceph/ceph/commit/3bc95dfc1b88 + - https://github.com/ceph/ceph/commit/77ff7c3dc6dd + - https://github.com/ceph/ceph/commit/1c5fea67 + (LP: #1529841) + + -- Alexey Sheplyakov Tue, 29 Dec 2015 16:52:25 +0300 + ceph (0.94.5-0u~u14.04+mos1) mos8.0; urgency=medium * Rebuild for Ubuntu 14.04 diff --git a/debian/patches/Compare-parted-output-with-the-dereferenced-path.patch b/debian/patches/Compare-parted-output-with-the-dereferenced-path.patch new file mode 100644 index 00000000..4084d253 --- /dev/null +++ b/debian/patches/Compare-parted-output-with-the-dereferenced-path.patch @@ -0,0 +1,30 @@ +From: Joe Julian +Date: Fri, 9 Oct 2015 12:57:06 -0700 +Subject: Compare parted output with the dereferenced path + +Compare parted output with the dereferenced path of the device as parted +prints that instead of the symlink we called it with. + +http://tracker.ceph.com/issues/13438 Fixes: #13438 + +Signed-off-by: Joe Julian +--- + src/ceph-disk | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/ceph-disk b/src/ceph-disk +index 1ff30b7..7620ff8 100755 +--- a/src/ceph-disk ++++ b/src/ceph-disk +@@ -996,9 +996,9 @@ def get_free_partition_index(dev): + 'BYT;' not in lines): + raise Error('parted output expected to contain one of ' + + 'CHH; CYL; or BYT; : ' + lines) +- if dev not in lines: ++ if os.path.realpath(dev) not in lines: + raise Error('parted output expected to contain ' + dev + ': ' + lines) +- _, partitions = lines.split(dev) ++ _, partitions = lines.split(os.path.realpath(dev)) + partition_numbers = extract_parted_partition_numbers(partitions) + if partition_numbers: + return max(partition_numbers) + 1 diff --git a/debian/patches/ceph-disk-is_mpath-predicate-for-multipath-devices.patch b/debian/patches/ceph-disk-is_mpath-predicate-for-multipath-devices.patch new file mode 100644 index 00000000..87b5f803 --- /dev/null +++ b/debian/patches/ceph-disk-is_mpath-predicate-for-multipath-devices.patch @@ -0,0 +1,89 @@ +From: Loic Dachary +Date: Mon, 17 Aug 2015 22:52:25 +0200 +Subject: ceph-disk: is_mpath predicate for multipath devices + +The is_mpath predicate returns True if a device is managed by +multipath. It is based on the devicemapper uuid content which is +expected to always contain the mpath- string to identify the multipath +subsystem. + +The block_path helper is added to convert the path to a device to the +/sys directory that describes it. It uses the major and minor number +instead of the device name because it is more reliable. The rationale +including an actual example is added as a comment for future +maintainers. + +Signed-off-by: Loic Dachary +--- + src/ceph-disk | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 51 insertions(+) + +diff --git a/src/ceph-disk b/src/ceph-disk +index 7620ff8..7163eca 100755 +--- a/src/ceph-disk ++++ b/src/ceph-disk +@@ -88,6 +88,7 @@ DMCRYPT_TOBE_UUID = '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be' + DMCRYPT_JOURNAL_TOBE_UUID = '89c57f98-2fe5-4dc0-89c1-35865ceff2be' + + DEFAULT_FS_TYPE = 'xfs' ++SYSFS = '/sys' + + MOUNT_OPTIONS = dict( + btrfs='noatime,user_subvol_rm_allowed', +@@ -358,6 +359,56 @@ def platform_information(): + str(codename).strip() + ) + ++# ++# An alternative block_path implementation would be ++# ++# name = basename(dev) ++# return /sys/devices/virtual/block/$name ++# ++# It is however more fragile because it relies on the fact ++# that the basename of the device the user will use always ++# matches the one the driver will use. On Ubuntu 14.04, for ++# instance, when multipath creates a partition table on ++# ++# /dev/mapper/353333330000007d0 -> ../dm-0 ++# ++# it will create partition devices named ++# ++# /dev/mapper/353333330000007d0-part1 ++# ++# which is the same device as /dev/dm-1 but not a symbolic ++# link to it: ++# ++# ubuntu@other:~$ ls -l /dev/mapper /dev/dm-1 ++# brw-rw---- 1 root disk 252, 1 Aug 15 17:52 /dev/dm-1 ++# lrwxrwxrwx 1 root root 7 Aug 15 17:52 353333330000007d0 -> ../dm-0 ++# brw-rw---- 1 root disk 252, 1 Aug 15 17:52 353333330000007d0-part1 ++# ++# Using the basename in this case fails. ++# ++def block_path(dev): ++ path = os.path.realpath(dev) ++ rdev = os.stat(path).st_rdev ++ (M, m) = (os.major(rdev), os.minor(rdev)) ++ return "{sysfs}/dev/block/{M}:{m}".format(sysfs=SYSFS, M=M, m=m) ++ ++def get_dm_uuid(dev): ++ uuid_path = os.path.join(block_path(dev), 'dm', 'uuid') ++ LOG.debug("get_dm_uuid " + dev + " uuid path is " + uuid_path) ++ if not os.path.exists(uuid_path): ++ return False ++ uuid = open(uuid_path, 'r').read() ++ LOG.debug("get_dm_uuid " + dev + " uuid is " + uuid) ++ return uuid ++ ++def is_mpath(dev): ++ """ ++ True if the path is managed by multipath ++ """ ++ uuid = get_dm_uuid(dev) ++ return (uuid and ++ (re.match('part\d+-mpath-', uuid) or ++ re.match('mpath-', uuid))) + + def get_dev_name(path): + """ diff --git a/debian/patches/ceph-disk-multipath-support-for-is_partition-and-lis.patch b/debian/patches/ceph-disk-multipath-support-for-is_partition-and-lis.patch new file mode 100644 index 00000000..253639cf --- /dev/null +++ b/debian/patches/ceph-disk-multipath-support-for-is_partition-and-lis.patch @@ -0,0 +1,143 @@ +From: Loic Dachary +Date: Mon, 17 Aug 2015 23:25:45 +0200 +Subject: ceph-disk: multipath support for is_partition and list_partitions + +The is_partition predicate and the list_partitions function support +devices managed by multipath. + +A set of helpers dedicated to multipath devices is implemented because +the content of the corresponding /sys directory does not use the same +conventions as regular devices regarding partitions. + +Instead of relying on subdirectories such as /sys/block/name/name1, the +devicemapper uuid file is used and expected to start with part\d+. The +holders/slaves directories provide pointers between the whole device and +the partition devices. + +Although these structural differences reduce the opportunity for +code factorization, it is easier for backward compatibility since the +multipath specific logic is limited to if is_mpath(dev) branches. + +http://tracker.ceph.com/issues/11881 Refs: #11881 + +Signed-off-by: Loic Dachary +--- + src/ceph-disk | 62 ++++++++++++++++++++++++++++++++++++++++++++++------------- + 1 file changed, 49 insertions(+), 13 deletions(-) + +diff --git a/src/ceph-disk b/src/ceph-disk +index 7163eca..218f92a 100755 +--- a/src/ceph-disk ++++ b/src/ceph-disk +@@ -469,6 +469,14 @@ def get_dev_size(dev, size='megabytes'): + os.close(fd) + + ++def get_partition_mpath(dev, pnum): ++ part_re = "part{pnum}-mpath-".format(pnum=pnum) ++ partitions = list_partitions_mpath(dev, part_re) ++ if partitions: ++ return partitions[0] ++ else: ++ return None ++ + def get_partition_dev(dev, pnum): + """ + get the device name for a partition +@@ -479,13 +487,16 @@ def get_partition_dev(dev, pnum): + sda 1 -> sda1 + cciss/c0d1 1 -> cciss!c0d1p1 + """ +- name = get_dev_name(os.path.realpath(dev)) + partname = None +- for f in os.listdir(os.path.join('/sys/block', name)): +- if f.startswith(name) and f.endswith(str(pnum)): +- # we want the shortest name that starts with the base name and ends with the partition number +- if not partname or len(f) < len(partname): +- partname = f ++ if is_mpath(dev): ++ partname = get_partition_mpath(dev, pnum) ++ else: ++ name = get_dev_name(os.path.realpath(dev)) ++ for f in os.listdir(os.path.join('/sys/block', name)): ++ if f.startswith(name) and f.endswith(str(pnum)): ++ # we want the shortest name that starts with the base name and ends with the partition number ++ if not partname or len(f) < len(partname): ++ partname = f + if partname: + return get_dev_path(partname) + else: +@@ -498,21 +509,40 @@ def list_all_partitions(): + """ + dev_part_list = {} + for name in os.listdir('/sys/block'): ++ LOG.debug("list_all_partitions: " + name) + # /dev/fd0 may hang http://tracker.ceph.com/issues/6827 + if re.match(r'^fd\d$', name): + continue +- if not os.path.exists(os.path.join('/sys/block', name, 'device')): +- continue +- dev_part_list[name] = list_partitions(name) ++ dev_part_list[name] = list_partitions(os.path.join('/dev', name)) + return dev_part_list + ++def list_partitions(dev): ++ dev = os.path.realpath(dev) ++ if is_mpath(dev): ++ return list_partitions_mpath(dev) ++ else: ++ return list_partitions_device(dev) ++ ++def list_partitions_mpath(dev, part_re="part\d+-mpath-"): ++ p = block_path(dev) ++ partitions = [] ++ holders = os.path.join(p, 'holders') ++ for holder in os.listdir(holders): ++ uuid_path = os.path.join(holders, holder, 'dm', 'uuid') ++ uuid = open(uuid_path, 'r').read() ++ LOG.debug("list_partitions_mpath: " + uuid_path + " uuid = " + uuid) ++ if re.match(part_re, uuid): ++ partitions.append(holder) ++ return partitions ++ + +-def list_partitions(basename): ++def list_partitions_device(dev): + """ + Return a list of partitions on the given device name + """ + partitions = [] +- for name in os.listdir(os.path.join('/sys/block', basename)): ++ basename = os.path.basename(dev) ++ for name in os.listdir(block_path(dev)): + if name.startswith(basename): + partitions.append(name) + return partitions +@@ -535,10 +565,17 @@ def get_partition_base(dev): + return '/dev/' + basename + raise Error('no parent device for partition', dev) + ++def is_partition_mpath(dev): ++ uuid = get_dm_uuid(dev) ++ return bool(re.match('part\d+-mpath-', uuid)) ++ + def is_partition(dev): + """ + Check whether a given device path is a partition or a full disk. + """ ++ if is_mpath(dev): ++ return is_partition_mpath(dev) ++ + dev = os.path.realpath(dev) + if not stat.S_ISBLK(os.lstat(dev).st_mode): + raise Error('not a block device', dev) +@@ -612,8 +649,7 @@ def verify_not_in_use(dev, check_partitions=False): + raise Error('Device %s is in use by a device-mapper mapping (dm-crypt?)' % dev, ','.join(holders)) + + if check_partitions and not is_partition(dev): +- basename = get_dev_name(os.path.realpath(dev)) +- for partname in list_partitions(basename): ++ for partname in list_partitions(dev): + partition = get_dev_path(partname) + if is_mounted(partition): + raise Error('Device is mounted', partition) diff --git a/debian/patches/ceph-disk-multipath-support-for-split_dev_base_partn.patch b/debian/patches/ceph-disk-multipath-support-for-split_dev_base_partn.patch new file mode 100644 index 00000000..589fdbfe --- /dev/null +++ b/debian/patches/ceph-disk-multipath-support-for-split_dev_base_partn.patch @@ -0,0 +1,61 @@ +From: Loic Dachary +Date: Mon, 17 Aug 2015 23:51:24 +0200 +Subject: ceph-disk: multipath support for split_dev_base_partnum + +split_dev_base_partnum returns the path of the whole disk in +/dev/mapper. The base variable name to designate the device for the +whole disk is a misnomer since it cannot be used as a basename to +rebuild the parition device name in the case of multipath. + +The logic of split_dev_base_partnum for devices is reworked to use +/sys/dev/block/M:m/partition instead of device name parsing. + +http://tracker.ceph.com/issues/11881 Refs: #11881 + +Signed-off-by: Loic Dachary +--- + src/ceph-disk | 23 +++++++++++++++++++---- + 1 file changed, 19 insertions(+), 4 deletions(-) + +diff --git a/src/ceph-disk b/src/ceph-disk +index 218f92a..394eb4a 100755 +--- a/src/ceph-disk ++++ b/src/ceph-disk +@@ -569,6 +569,18 @@ def is_partition_mpath(dev): + uuid = get_dm_uuid(dev) + return bool(re.match('part\d+-mpath-', uuid)) + ++def partnum_mpath(dev): ++ uuid = get_dm_uuid(dev) ++ return re.findall('part(\d+)-mpath-', uuid)[0] ++ ++def get_partition_base_mpath(dev): ++ slave_path = os.path.join(block_path(dev), 'slaves') ++ slaves = os.listdir(slave_path) ++ assert slaves ++ name_path = os.path.join(slave_path, slaves[0], 'dm', 'name') ++ name = open(name_path, 'r').read().strip() ++ return os.path.join('/dev/mapper', name) ++ + def is_partition(dev): + """ + Check whether a given device path is a partition or a full disk. +@@ -2449,11 +2461,14 @@ def get_dev_fs(dev): + + + def split_dev_base_partnum(dev): +- if 'loop' in dev or 'cciss' in dev or 'nvme' in dev: +- return re.match('(.*\d+)p(\d+)', dev).group(1, 2) ++ if is_mpath(dev): ++ partnum = partnum_mpath(dev) ++ base = get_partition_base_mpath(dev) + else: +- return re.match('(\D+)(\d+)', dev).group(1, 2) +- ++ b = block_path(dev) ++ partnum = open(os.path.join(b, 'partition')).read().strip() ++ base = get_partition_base(dev) ++ return (base, partnum) + + def get_partition_type(part): + """ diff --git a/debian/patches/series b/debian/patches/series index 44e9de69..c7a88511 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -1,5 +1,9 @@ ## Backported / Upstream sleep-recover.patch +Compare-parted-output-with-the-dereferenced-path.patch +ceph-disk-is_mpath-predicate-for-multipath-devices.patch +ceph-disk-multipath-support-for-is_partition-and-lis.patch +ceph-disk-multipath-support-for-split_dev_base_partn.patch ## Debian rbdmap3-lazyumount.patch -- 2.45.2