Posted on | In |

命令格式

Ceph有个daemonperf工具，结合Ceph daemon的asok文件，可以检查Ceph各个组件的当前状态。

命令格式：ceph daemonperf

示例如下：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
# ceph daemonperf /var/run/ceph/ceph-osd.0.asok 2
---objecter--- -----------osd-----------
writ read actv|recop rd   wr   lat  ops |
  0    0    0 |   0   12k   0    0    1
  0    0    0 |   0  532k   0    0   62
  0    0    0 |   0  106k   0    0   10
  0    0    0 |   0  487k 4.2M   1   71

# ceph daemonperf /var/run/ceph/ceph-mds.mds-ceph0.asok 2
-----mds------ --mds_server-- ---objecter--- -----mds_cache----- ---mds_log----
rlat inos caps|hsr  hcs  hcr |writ read actv|recd recy stry purg|segs evts subm|
  0  137k  22k|  0    0   81 |  4    0  129 |  0    0  108k   4 | 54   40k  43
  0  139k  15k|  0    0  7.5k|941    1  132 |  0    0  111k 918 | 46   34k 4.7k
  0  145k  16k|  0    0   11k|592    0  129 |  0    0  116k 581 | 54   40k 6.3k
  0  123k 7.3k|  0    1  6.9k|231    1  129 |  0    0  119k 214 | 48   33k 3.7k
  0  128k 8.2k|  0    0  9.0k|509    0  130 |  0    0  123k 503 | 54   38k 5.0k
  1  129k 8.3k|  0    0  924 |405    1  128 |  0    0  123k 402 | 55   39k 867

源码实现

文件：ceph.in

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
def main():
    ...
    if parsed_args.admin_socket:
        sockpath = parsed_args.admin_socket
    elif len(childargs) > 0 and childargs[0] in ["daemon", "daemonperf"]:
        daemon_perf = (childargs[0] == "daemonperf")
        # Treat "daemon <path>" or "daemon <name>" like --admin_daemon <path>
        # Handle "daemonperf <path>" the same but requires no trailing args
        require_args = 2 if daemon_perf else 3
    ...
    if sockpath and daemon_perf:
        interval = 1
        count = None
        if len(childargs) > 0:
            try:
                interval = float(childargs[0])
                if interval < 0:
                    raise ValueError
            except ValueError:
                print('daemonperf: interval should be a positive number', file=sys.stderr)
                return errno.EINVAL
        if len(childargs) > 1:
            if not childargs[1].isdigit():
                print('daemonperf: count should be a positive integer', file=sys.stderr)
                return errno.EINVAL
            count = int(childargs[1])
        DaemonWatcher(sockpath).run(interval, count)
        return 0
	...

文件：pybind/ceph_daemon.py

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
class DaemonWatcher(object):
    """
    Given a Ceph daemon's admin socket path, poll its performance counters
    and output a series of output lines showing the momentary values of
    counters of interest (those with the 'nick' property in Ceph's schema)
    """
    ...
    def _load_schema(self):
        """
        Populate our instance-local copy of the daemon's performance counter
        schema, and work out which stats we will display.
        """
        self._schema = json.loads(admin_socket(self.asok_path, ["perf", "schema"]))

        # Build list of which stats we will display, based on which
        # stats have a nickname
        self._stats = defaultdict(dict)
        for section_name, section_stats in self._schema.items():
            for name, schema_data in section_stats.items():
                if schema_data.get('nick'):
                    self._stats[section_name][name] = schema_data['nick']

    def run(self, interval, count=None, ostr=sys.stdout):
        """
        Print output at regular intervals until interrupted.

        :param ostr: Stream to which to send output
        """

        self._load_schema()
        self._colored = self.supports_color(ostr)

        self._print_headers(ostr)

        last_dump = json.loads(admin_socket(self.asok_path, ["perf", "dump"]))
        rows_since_header = 0
        term_height = 25

        try:
            while True:
                dump = json.loads(admin_socket(self.asok_path, ["perf", "dump"]))
                if rows_since_header > term_height - 2:
                    self._print_headers(ostr)
                    rows_since_header = 0
                self._print_vals(ostr, dump, last_dump)
                if count is not None:
                    count -= 1
                    if count <= 0:
                        break
                rows_since_header += 1
                last_dump = dump
                time.sleep(interval)
        except KeyboardInterrupt:
            return

所以ceph daemonperf <asok>命令的输出是根据ceph daemon <asok> perf dump/schema的输出整理的。

实现中存了上一次的perf dump结果，所以这里获取的值是interval里的数据统计。

以ceph mds asok的perf dump/schema输出为例，看看每个项是什么含义：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
ceph daemon <mds-asok> perf schema
{
    "mds": {
        "reply_latency": {
            "type": 5,
            "description": "Reply latency",
            "nick": "rlat"
        },
        ...
        "inodes": {
            "type": 2,
            "description": "Inodes",
            "nick": "inos"
        },
        ...
        "caps": {
            "type": 2,
            "description": "Capabilities",
            "nick": "caps"
        },
        ...
    },
    "mds_cache": {
        "num_strays": {
            "type": 2,
            "description": "Stray dentries",
            "nick": "stry"
        },
        ...
        "strays_purged": {
            "type": 10,
            "description": "Stray dentries purged",
            "nick": "purg"
        },
    	...
        "num_recovering_enqueued": {
            "type": 2,
            "description": "Files waiting for recovery",
            "nick": "recy"
        },
        ...
        "recovery_completed": {
            "type": 10,
            "description": "File recoveries completed",
            "nick": "recd"
        }
    },
    "mds_log": {
        "evadd": {
            "type": 10,
            "description": "Events submitted",
            "nick": "subm"
        },
        ...
        "ev": {
            "type": 2,
            "description": "Events",
            "nick": "evts"
        },
        ...
        "seg": {
            "type": 2,
            "description": "Segments",
            "nick": "segs"
        },
        ...
    },
    "mds_server": {
        "handle_client_request": {
            "type": 10,
            "description": "Client requests",
            "nick": "hcr"
        },
        "handle_slave_request": {
            "type": 10,
            "description": "Slave requests",
            "nick": "hsr"
        },
        "handle_client_session": {
            "type": 10,
            "description": "Client session messages",
            "nick": "hcs"
        },
        ...
    },
    "objecter": {
        "op_active": {
            "type": 2,
            "description": "Operations active",
            "nick": "actv"
        },
        "op_r": {
            "type": 10,
            "description": "Read operations",
            "nick": "read"
        },
        "op_w": {
            "type": 10,
            "description": "Write operations",
            "nick": "writ"
        },
        ...
    },
}

schema输出里的含义：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
文件：common/perf_counters.h

enum perfcounter_type_d
{
    PERFCOUNTER_NONE = 0,
    PERFCOUNTER_TIME = 0x1,
    PERFCOUNTER_U64 = 0x2,
    PERFCOUNTER_LONGRUNAVG = 0x4,
    PERFCOUNTER_COUNTER = 0x8,
};

class PerfCounters
{
    /** Represents a PerfCounters data element. */
    struct perf_counter_data_any_d {
        ...
        const char *name;
        const char *description;
        const char *nick;
        enum perfcounter_type_d type;
        atomic64_t u64;
        atomic64_t avgcount;
        atomic64_t avgcount2;
        ...
    };
    ...
};

ceph daemon perf dump中与daemonperf相关的输出项：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
ceph daemon <mds-asok> perf dump
{
    "mds": {
    	...
        "reply_latency": {
            "avgcount": 1879241,
            "sum": 563.417555661
        },
        ...
        "inodes": 510943,
        ...
        "caps": 68002,
        ...
    },
    "mds_cache": {
        "num_strays": 415633,
        ...
        "strays_purged": 1192320,
        ...
        "num_recovering_enqueued": 0,
        ...
        "recovery_completed": 9
    },
    "mds_log": {
        "evadd": 2094629,
        ...
        "ev": 42664,
        ...
        "seg": 57,
        ...
    },
    "mds_server": {
        "handle_client_request": 1879915,
        "handle_slave_request": 0,
        "handle_client_session": 698,
        ...
    },
    "objecter": {
        "op_active": 64,
        ...
        "op_r": 832,
        "op_w": 1219634,
        ...
    },
}

Posted on | In |

概述

在Ceph的环境中，我们通常会使用SSD来作为OSD的Journal，而OSD的数据盘是普通的SATA盘，在实践中，经常会发现SATA盘的性能瓶颈影响了OSD的性能，那能不能继续压榨SSD的性能来提升OSD的性能呢？

答案是肯定的，可以使用SSD加速SATA盘的策略来加速作为OSD数据盘的SATA盘，通常的策略有：

  • flashcache
  • bcache

有文章对比测试过这两种cache策略的性能，bcache的性能会好很多，这里介绍如何使用bcache来给OSD加速。

测试环境

Ceph版本：Jewel 10.2.9

作为Ceph机器的物理机的磁盘配置如下：

  1. SSD - 745 G，三块
  2. SATA - 3.7 T，九块

对磁盘规划如下：

  1. 每个SSD分出3个10G分区，作为三个SATA盘OSD的journal
  2. 每个SSD剩余空间分为1个分区，使用bcache来加速三个SATA盘

磁盘分区前如下：

1
2
3
4
5
6
root@ceph0:~/yangguanjun# lsblk
NAME   MAJ:MIN RM   SIZE RO TYPE MOUNTPOINT
...
sdb      8:16   0 745.2G  0 disk
sdc      8:32   0 745.2G  0 disk
sdd      8:48   0 745.2G  0 disk

磁盘分区后为：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
root@ceph0:~# lsblk
NAME        MAJ:MIN RM   SIZE RO TYPE MOUNTPOINT
...
sdb           8:16   0 745.2G  0 disk
├─sdb1        8:17   0    10G  0 part
├─sdb2        8:18   0    10G  0 part
├─sdb3        8:19   0    10G  0 part
└─sdb4        8:20   0 715.2G  0 part
sdc           8:32   0 745.2G  0 disk
├─sdc1        8:33   0    10G  0 part
├─sdc2        8:34   0    10G  0 part
├─sdc3        8:35   0    10G  0 part
└─sdc4        8:36   0 715.2G  0 part
sdd           8:48   0 745.2G  0 disk
├─sdd1        8:49   0    10G  0 part
├─sdd2        8:50   0    10G  0 part
├─sdd1        8:51   0    10G  0 part
└─sdd4        8:52   0 715.2G  0 part

设备性能

针对当前环境，先要了解下各个硬件的性能，通过fio测试结果如下：

磁盘类型 read write randread randwrite
SATA 155MB/s 158MB/s 126 219
SSD 508MB/s 426MB/s 69.2k 45.5k

参考文章配置SATA盘与SSD盘的bcache策略：http://www.yangguanjun.com/2018/03/26/lvm-sata-ssd-bcache/

配置bcache命令如下：

1
# make-bcache -B /dev/sde -C /dev/sdc4

配置后的块设备信息如下：

1
2
3
4
5
6
7
8
9
10
11
# lsblk
...
sdc           8:32   0 745.2G  0 disk
├─sdc1        8:33   0    10G  0 part
├─sdc2        8:34   0    10G  0 part
├─sdc3        8:35   0    10G  0 part
└─sdc4        8:36   0 715.2G  0 part
  ├─bcache0 253:0    0   3.7T  0 disk
...
sde           8:64   0   3.7T  0 disk
└─bcache0   253:0    0   3.7T  0 disk

之后测试bcache加速设备性能：

bcache的不同缓存策略 randwrite
bcache [writethrough] 218
bcache [writeback] 38.8k

上面结果看出 bcache 配置为writeback模式后，加速设备性能很高，会比SSD盘的性能略差些。但之后数据会回刷到SATA上，通过iostat可以看到SATA盘会繁忙好一阵子。

部署OSD

默认直接使用ceph-deploy部署bcache设备时会报错，如下：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
# ceph-deploy osd prepare ceph0:/dev/bcache0:/dev/sdd1
[ceph_deploy.conf][DEBUG ] found configuration file at: /root/.cephdeploy.conf
[ceph_deploy.cli][INFO  ] Invoked (1.5.37): /usr/bin/ceph-deploy osd prepare ceph0:/dev/bcache0:/dev/sdd1
[ceph_deploy.cli][INFO  ] ceph-deploy options:
[ceph_deploy.cli][INFO  ]  username                      : None
[ceph_deploy.cli][INFO  ]  disk                          : [('ceph0', '/dev/bcache0', '/dev/sdd1')]
[ceph_deploy.cli][INFO  ]  dmcrypt                       : False
[ceph_deploy.cli][INFO  ]  verbose                       : False
[ceph_deploy.cli][INFO  ]  bluestore                     : None
[ceph_deploy.cli][INFO  ]  overwrite_conf                : False
[ceph_deploy.cli][INFO  ]  subcommand                    : prepare
[ceph_deploy.cli][INFO  ]  dmcrypt_key_dir               : /etc/ceph/dmcrypt-keys
[ceph_deploy.cli][INFO  ]  quiet                         : False
[ceph_deploy.cli][INFO  ]  cd_conf                       : <ceph_deploy.conf.cephdeploy.Conf instance at 0x7f81bab7b200>
[ceph_deploy.cli][INFO  ]  cluster                       : ceph
[ceph_deploy.cli][INFO  ]  fs_type                       : xfs
[ceph_deploy.cli][INFO  ]  func                          : <function osd at 0x7f81bab6a938>
[ceph_deploy.cli][INFO  ]  ceph_conf                     : None
[ceph_deploy.cli][INFO  ]  default_release               : False
[ceph_deploy.cli][INFO  ]  zap_disk                      : False
[ceph_deploy.osd][DEBUG ] Preparing cluster ceph disks ceph0:/dev/bcache0:/dev/sdd1
[ceph0][DEBUG ] connected to host: ceph0
[ceph0][DEBUG ] detect platform information from remote host
[ceph0][DEBUG ] detect machine type
[ceph0][DEBUG ] find the location of an executable
[ceph_deploy.osd][INFO  ] Distro info: CentOS Linux 7.4.1708 Core
[ceph_deploy.osd][DEBUG ] Deploying osd to ceph0
[ceph0][DEBUG ] write cluster configuration to /etc/ceph/{cluster}.conf
[ceph_deploy.osd][DEBUG ] Preparing host ceph0 disk /dev/bcache0 journal /dev/sdd1 activate False
[ceph0][DEBUG ] find the location of an executable
[ceph0][INFO  ] Running command: /usr/sbin/ceph-disk -v prepare --cluster ceph --fs-type xfs -- /dev/bcache0 /dev/sdd1
[ceph0][WARNIN] command: Running command: /usr/bin/ceph-osd --cluster=ceph --show-config-value=fsid
[ceph0][WARNIN] command: Running command: /usr/bin/ceph-osd --check-allows-journal -i 0 --log-file $run_dir/$cluster-osd-check.log --cluster ceph --setuser ceph --setgroup ceph
[ceph0][WARNIN] command: Running command: /usr/bin/ceph-osd --check-wants-journal -i 0 --log-file $run_dir/$cluster-osd-check.log --cluster ceph --setuser ceph --setgroup ceph
[ceph0][WARNIN] command: Running command: /usr/bin/ceph-osd --check-needs-journal -i 0 --log-file $run_dir/$cluster-osd-check.log --cluster ceph --setuser ceph --setgroup ceph
[ceph0][WARNIN] get_dm_uuid: get_dm_uuid /dev/bcache0 uuid path is /sys/dev/block/252:8/dm/uuid
[ceph0][WARNIN] command: Running command: /usr/bin/ceph-osd --cluster=ceph --show-config-value=osd_journal_size
[ceph0][WARNIN] get_dm_uuid: get_dm_uuid /dev/bcache0 uuid path is /sys/dev/block/252:8/dm/uuid
[ceph0][WARNIN] get_dm_uuid: get_dm_uuid /dev/bcache0 uuid path is /sys/dev/block/252:8/dm/uuid
[ceph0][WARNIN] get_dm_uuid: get_dm_uuid /dev/bcache0 uuid path is /sys/dev/block/252:8/dm/uuid
[ceph0][WARNIN] command: Running command: /usr/bin/ceph-conf --cluster=ceph --name=osd. --lookup osd_mkfs_options_xfs
[ceph0][WARNIN] command: Running command: /usr/bin/ceph-conf --cluster=ceph --name=osd. --lookup osd_fs_mkfs_options_xfs
[ceph0][WARNIN] command: Running command: /usr/bin/ceph-conf --cluster=ceph --name=osd. --lookup osd_mount_options_xfs
[ceph0][WARNIN] command: Running command: /usr/bin/ceph-conf --cluster=ceph --name=osd. --lookup osd_fs_mount_options_xfs
[ceph0][WARNIN] get_dm_uuid: get_dm_uuid /dev/sdd1 uuid path is /sys/dev/block/8:51/dm/uuid
[ceph0][WARNIN] prepare_device: Journal /dev/sdd1 is a partition
[ceph0][WARNIN] get_dm_uuid: get_dm_uuid /dev/sdd1 uuid path is /sys/dev/block/8:51/dm/uuid
[ceph0][WARNIN] prepare_device: OSD will not be hot-swappable if journal is not the same device as the osd data
[ceph0][WARNIN] command: Running command: /usr/sbin/blkid -o udev -p /dev/sdd1
[ceph0][WARNIN] prepare_device: Journal /dev/sdd1 was not prepared with ceph-disk. Symlinking directly.
[ceph0][WARNIN] get_dm_uuid: get_dm_uuid /dev/bcache0 uuid path is /sys/dev/block/252:8/dm/uuid
[ceph0][WARNIN] set_data_partition: Creating osd partition on /dev/bcache0
[ceph0][WARNIN] get_dm_uuid: get_dm_uuid /dev/bcache0 uuid path is /sys/dev/block/252:8/dm/uuid
[ceph0][WARNIN] ptype_tobe_for_name: name = data
[ceph0][WARNIN] get_dm_uuid: get_dm_uuid /dev/bcache0 uuid path is /sys/dev/block/252:8/dm/uuid
[ceph0][WARNIN] create_partition: Creating data partition num 1 size 0 on /dev/bcache0
[ceph0][WARNIN] command_check_call: Running command: /usr/sbin/sgdisk --largest-new=1 --change-name=1:ceph data --partition-guid=1:cf599e47-c299-46c1-a385-aff4b0d25f1f --typecode=1:89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be --mbrtogpt -- /dev/bcache0
[ceph0][WARNIN] Caution: invalid main GPT header, but valid backup; regenerating main header
[ceph0][WARNIN] from backup!
[ceph0][WARNIN]
[ceph0][WARNIN] Invalid partition data!
[ceph0][WARNIN] '/usr/sbin/sgdisk --largest-new=1 --change-name=1:ceph data --partition-guid=1:cf599e47-c299-46c1-a385-aff4b0d25f1f --typecode=1:89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be --mbrtogpt -- /dev/bcache0' failed with status code 2
[ceph0][ERROR ] RuntimeError: command returned non-zero exit status: 1
[ceph_deploy.osd][ERROR ] Failed to execute command: /usr/sbin/ceph-disk -v prepare --cluster ceph --fs-type xfs -- /dev/bcache0 /dev/sdd1
[ceph_deploy][ERROR ] GenericError: Failed to create 1 OSDs

搜索有如下参考，ceph-deploy还不支持bcache设备，默认ceph-deploy会尝试对bcache设备进行分区，而bcache设备是不支持分区后挂载的，所以会导致命令失败。

http://tracker.ceph.com/issues/13278

https://github.com/ceph/ceph/pull/16327

解决办法一

修改ceph-disk的代码，把上述链接中的patch加上，可惜没搞成功，很奇怪修改后的代码貌似没跑到。。。因时间紧急就没再研究，但这个办法肯定是可行的！

解决办法二

手动格式化bcache设备，mount上后，通过ceph-deploy指定目录来部署了，步骤如下：

1
2
3
# mkdir /var/lib/ceph/osd/ceph-0
# mkfs.xfs /dev/bcache0
# mount /dev/bcache0 /var/lib/ceph/osd/ceph-0

然后再尝试部署，报错如下：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
# ceph-deploy --overwrite-conf osd prepare ceph0:/var/lib/ceph/osd/ceph-0:/dev/sdd1
...
[ceph0][WARNIN] command: Running command: /usr/bin/timeout 300 ceph-osd --cluster ceph --mkfs --mkkey -i 195 --monmap /var/lib/ceph/osd/ceph-0/activate.monmap --osd-data /var/lib/ceph/osd/ceph-0 --osd-journal /var/lib/ceph/osd/ceph-0/journal --osd-uuid 9ff0983b-74e2-4fc1-8ba8-cfb688024284 --keyring /var/lib/ceph/osd/ceph-0/keyring --setuser ceph --setgroup ceph
[ceph0][WARNIN] Traceback (most recent call last):
[ceph0][WARNIN]   File "/usr/sbin/ceph-disk", line 9, in <module>
[ceph0][WARNIN]     load_entry_point('ceph-disk==1.0.0', 'console_scripts', 'ceph-disk')()
[ceph0][WARNIN]   File "/usr/lib/python2.7/site-packages/ceph_disk/main.py", line 5553, in run
[ceph0][WARNIN]     main(sys.argv[1:])
[ceph0][WARNIN]   File "/usr/lib/python2.7/site-packages/ceph_disk/main.py", line 5504, in main
[ceph0][WARNIN]     args.func(args)
[ceph0][WARNIN]   File "/usr/lib/python2.7/site-packages/ceph_disk/main.py", line 3631, in main_activate
[ceph0][WARNIN]     init=args.mark_init,
[ceph0][WARNIN]   File "/usr/lib/python2.7/site-packages/ceph_disk/main.py", line 3451, in activate_dir
[ceph0][WARNIN]     (osd_id, cluster) = activate(path, activate_key_template, init)
[ceph0][WARNIN]   File "/usr/lib/python2.7/site-packages/ceph_disk/main.py", line 3556, in activate
[ceph0][WARNIN]     keyring=keyring,
[ceph0][WARNIN]   File "/usr/lib/python2.7/site-packages/ceph_disk/main.py", line 3010, in mkfs
[ceph0][WARNIN]     '--setgroup', get_ceph_group(),
[ceph0][WARNIN]   File "/usr/lib/python2.7/site-packages/ceph_disk/main.py", line 2957, in ceph_osd_mkfs
[ceph0][WARNIN]     raise Error('%s failed : %s' % (str(arguments), error))
[ceph0][WARNIN] ceph_disk.main.Error: Error: ['ceph-osd', '--cluster', 'ceph', '--mkfs', '--mkkey', '-i', u'195', '--monmap', '/var/lib/ceph/osd/ceph-0/activate.monmap', '--osd-data', '/var/lib/ceph/osd/ceph-0', '--osd-journal', '/var/lib/ceph/osd/ceph-0/journal', '--osd-uuid', u'9ff0983b-74e2-4fc1-8ba8-cfb688024284', '--keyring', '/var/lib/ceph/osd/ceph-0/keyring', '--setuser', 'ceph', '--setgroup', 'ceph'] failed : parse error setting 'osd_deep_scrub_interval' to '2592000 // every mouth'
[ceph0][WARNIN] 2018-04-26 16:02:27.180328 7f3085e23940 -1 filestore(/var/lib/ceph/osd/ceph-0) mkfs: write_version_stamp() failed: (13) Permission denied
[ceph0][WARNIN] 2018-04-26 16:02:27.180346 7f3085e23940 -1 OSD::mkfs: ObjectStore::mkfs failed with error -13
[ceph0][WARNIN] 2018-04-26 16:02:27.180441 7f3085e23940 -1  ** ERROR: error creating empty object store in /var/lib/ceph/osd/ceph-0: (13) Permission denied
[ceph0][WARNIN]
[ceph0][ERROR ] RuntimeError: command returned non-zero exit status: 1
[ceph_deploy][ERROR ] RuntimeError: Failed to execute command: /usr/sbin/ceph-disk -v activate --mark-init systemd --mount /var/lib/ceph/osd/ceph-0

从输出里看是权限的问题，执行如下两条命令后，部署成功：

1
2
root@ceph0:/var/lib/ceph/osd# chown -R ceph:ceph /dev/sdd1
root@ceph0:/var/lib/ceph/osd# chown -R ceph:ceph ceph-0

Ceph OSD开机自启动

bcache开机启动

添加bcache开机启动

1
2
3
4
5
6
7
8
9
# cat /etc/sysconfig/modules/bcache.modules
#! /bin/sh

/sbin/modinfo -F filename bcache > /dev/null 2>&1
if [ $? -eq 0 ]; then
    /sbin/modprobe -f bcache
fi

# chmod 755 /etc/sysconfig/modules/bcache.modules

自动挂载OSD目录

添加磁盘自动挂载，保证重启后Ceph OSD能自动运行

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
root@ceph0:~# blkid
/dev/sdd1: PARTUUID="81732128-2073-4d93-8582-377f4f9a701f"
...
/dev/bcache0: UUID="88ef6ba6-fd13-478c-8b56-82951183f1d3" TYPE="xfs"

root@ceph0:~# cat /etc/fstab
#
# /etc/fstab
# Created by anaconda on Wed May 24 15:20:03 2017
#
# Accessible filesystems, by reference, are maintained under '/dev/disk'
# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info
#
LABEL=/  /                       ext4    defaults        1 1
UUID=88ef6ba6-fd13-478c-8b56-82951183f1d3 /var/lib/ceph/osd/ceph-0 xfs defaults 0 2

Posted on | In |

需求

Ceph Luminous已经发布到12.2.4版本，经历了前几个版本的磨炼，Luminous版本也越来越稳定，它的Bluestore属性和CephFS提供的多主MDS一直是我们关注的重点。

参考ceph和ceph-deploy官方文档，本文介绍下使用ceph-deploy部署最新版Ceph Luminous 12.2.4，以及部署中遇到的问题。

软件版本

ceph-deploy版本

1
2
# ceph-deploy --version
2.0.0

ceph-deploy 2.0.0的changelog：

1
2
3
4
5
6
7
8
2.0.0
16-Jan-2018

- Backward incompatible API changes for OSD creation - will use ceph-volume and no longer consume ceph-disk.
- Remove python-distribute dependency
- Use /etc/os-release as a fallback when linux_distribution() doesn’t work
- Drop dmcrypt support (unsupported by ceph-volume for now)
- Allow debug modes for ceph-volume

参考：http://docs.ceph.com/ceph-deploy/docs/changelog.html#id1

系统版本

1
2
3
4
5
6
# lsb_release -a
LSB Version:	:core-4.1-amd64:core-4.1-noarch
Distributor ID:	CentOS
Description:	CentOS Linux release 7.3.1611 (Core)
Release:	7.3.1611
Codename:	Core

Ceph版本

yum源配置，选择最新的Luminous版本：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# cat /etc/yum.repos.d/ceph.repo
[ceph]
name=Ceph packages for $basearch
baseurl=http://mirrors.163.com/ceph/rpm-luminous/el7/$basearch
enabled=1
priority=2
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.asc

[ceph-noarch]
name=Ceph noarch packages
baseurl=http://mirrors.163.com/ceph/rpm-luminous/el7/noarch
enabled=1
priority=2
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.asc

[ceph-source]
name=Ceph source packages
baseurl=http://mirrors.163.com/ceph/rpm-luminous/el7/SRPMS
enabled=0
priority=2
gpgcheck=1
gpgkey=https://download.ceph.com/keys/release.asc

ceph安装

1
2
3
4
5
6
# yum install -y ceph ceph-radosgw
or
# ceph-deploy install [hosts]

# ceph -v
ceph version 12.2.4 (52085d5249a80c5f5121a76d6288429f35e4e77b) luminous (stable)

准备工作

使用ceph-deploy开始部署前，有如下几点要提前做好

  1. 部署节点安装 ceph-deploy

    1
    # yum install -y ceph-deploy

  2. 部署节点与ceph nodes之间ssh无密码访问

    1
    2
    # ssh-keygen
    # cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

  3. ceph nodes的hostname配置

    1
    # vim /etc/hostname

  4. 部署节点/etc/hosts配置 ceph nodes的hostname与ip的对应

    1
    2
    3
    4
    # vim /etc/hosts
    100.60.0.20 ceph0
    100.60.0.21 ceph1
    100.60.0.22 ceph2

  5. ceph nodes配置ntp server

    1
    2
    # yum install -y ntp ntpdate ntp-doc
    # systemctl start ntpd

  6. ceph nodes安装ssh server

    1
    # yum install -y openssh-server

  7. ceph nodes关闭或配置防火墙

    1
    2
    # systemctl stop firewalld
    # systemctl disable firewalld

    ​

Ceph Monitor部署

开始部署Ceph Cluster，创建三个monitors：

1
2
# ceph-deploy new ceph0 ceph1 ceph2
# ceph-deploy mon create ceph0 ceph1 ceph2

创建ceph keys：

1
# ceph-deploy gatherkeys ceph0 ceph1 ceph2

一定要开启ceph 认证，不然该命令执行失败

与ceph-deploy mon create命令执行完要有一定时间间隔，等到monitor集群正常了才能执行

分发ceph配置和admin key到ceph集群节点：

1
# ceph-deploy admin ceph0 ceph1 ceph2

检查当前ceph集群状态：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# ceph -s
  cluster:
    id:     5b2192ff-299c-4024-9f10-93af008e66d3
    health: HEALTH_OK

  services:
    mon: 3 daemons, quorum ceph0,ceph2,ceph1
    mgr: no daemons active
    osd: 0 osds: 0 up, 0 in

  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 bytes
    usage:   0 kB used, 0 kB / 0 kB avail
    pgs:

参考：http://docs.ceph.com/ceph-deploy/docs/index.html#creating-a-new-configuration

Ceph manager部署

部署ceph-mgr组件：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# ceph-deploy mgr create ceph0 ceph1 ceph2
# ceph -s
  cluster:
    id:     5b2192ff-299c-4024-9f10-93af008e66d3
    health: HEALTH_OK

  services:
    mon: 3 daemons, quorum ceph0,ceph2,ceph1
    mgr: ceph0(active), standbys: ceph2, ceph1
    osd: 0 osds: 0 up, 0 in

  data:
    pools:   0 pools, 0 pgs
    objects: 0 objects, 0 bytes
    usage:   0 kB used, 0 kB / 0 kB avail
    pgs:

参考：http://docs.ceph.com/docs/master/mgr/

Ceph Luminous主推的用于Ceph集群管理的组件，默认在所有部署ceph-mon的节点都启动一个ceph-mgr

Ceph OSD部署

使用ceph-deploy工具来部署ceph osd：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
# ceph-deploy osd create -h
usage: ceph-deploy osd create [-h] [--data DATA] [--journal JOURNAL]
                              [--zap-disk] [--fs-type FS_TYPE] [--dmcrypt]
                              [--dmcrypt-key-dir KEYDIR] [--filestore]
                              [--bluestore] [--block-db BLOCK_DB]
                              [--block-wal BLOCK_WAL] [--debug]
                              [HOST]

positional arguments:
  HOST                  Remote host to connect

optional arguments:
  -h, --help            show this help message and exit
  --data DATA           The OSD data logical volume (vg/lv) or absolute path
                        to device
  --journal JOURNAL     Logical Volume (vg/lv) or path to GPT partition
  --zap-disk            DEPRECATED - cannot zap when creating an OSD
  --fs-type FS_TYPE     filesystem to use to format DEVICE (xfs, btrfs)
  --dmcrypt             use dm-crypt on DEVICE
  --dmcrypt-key-dir KEYDIR
                        directory where dm-crypt keys are stored
  --filestore           filestore objectstore
  --bluestore           bluestore objectstore
  --block-db BLOCK_DB   bluestore block.db path
  --block-wal BLOCK_WAL
                        bluestore block.wal path
  --debug               Enable debug mode on remote ceph-volume calls

注：从Ceph Luminous 12.2.2开始， ceph-disk 就被弃用了，开始使用新的工具ceph-volume。

参考：http://docs.ceph.com/docs/master/ceph-volume/

创建一个bluestore的osd，有以下几种设备选择：

  • A block device, a block.wal, and a block.db device
  • A block device and a block.wal device
  • A block device and a block.db device
  • A single block device

参考：http://docs.ceph.com/docs/master/ceph-volume/lvm/prepare/#bluestore

block device也有如下三种选项：

  • 整块磁盘
  • 磁盘分区
  • 逻辑卷（a logical volume of LVM）

配置使用整块磁盘时，ceph-volume会自动创建一个logical volume使用

单独块设备创建OSD

整块磁盘

命令格式：

1
# ceph-deploy osd create [host] --data [/path/to/device]

首先销毁磁盘的分区信息：

1
2
3
4
5
6
7
8
9
10
11
12
13
[root@ceph0 ceph-deploy]# ceph-deploy disk zap ceph0 /dev/sdb
...
[ceph_deploy][ERROR ] Traceback (most recent call last):
[ceph_deploy][ERROR ]   File "/usr/lib/python2.7/site-packages/ceph_deploy/util/decorators.py", line 69, in newfunc
[ceph_deploy][ERROR ]     return f(*a, **kw)
[ceph_deploy][ERROR ]   File "/usr/lib/python2.7/site-packages/ceph_deploy/cli.py", line 164, in _main
[ceph_deploy][ERROR ]     return args.func(args)
[ceph_deploy][ERROR ]   File "/usr/lib/python2.7/site-packages/ceph_deploy/osd.py", line 438, in disk
[ceph_deploy][ERROR ]     disk_zap(args)
[ceph_deploy][ERROR ]   File "/usr/lib/python2.7/site-packages/ceph_deploy/osd.py", line 336, in disk_zap
[ceph_deploy][ERROR ]     if args.debug:
[ceph_deploy][ERROR ] AttributeError: 'Namespace' object has no attribute 'debug'
[ceph_deploy][ERROR ]

修改osd.py如下：

1
2
3
[root@ceph0 ceph-deploy]# vim /usr/lib/python2.7/site-packages/ceph_deploy/osd.py
        #if args.debug:
        if False:

再尝试disk zap成功：

1
2
3
4
5
6
7
8
9
10
11
12
13
[root@ceph0 ceph-deploy]# ceph-deploy disk zap ceph0 /dev/sdb
...
[ceph0][INFO  ] Running command: /usr/sbin/ceph-volume lvm zap /dev/sdb
[ceph0][DEBUG ] --> Zapping: /dev/sdb
[ceph0][DEBUG ] Running command: cryptsetup status /dev/mapper/
[ceph0][DEBUG ]  stdout: /dev/mapper/ is inactive.
[ceph0][DEBUG ] Running command: wipefs --all /dev/sdb
[ceph0][DEBUG ] Running command: dd if=/dev/zero of=/dev/sdb bs=1M count=10
[ceph0][DEBUG ]  stderr: 10+0 records in
[ceph0][DEBUG ] 10+0 records out
[ceph0][DEBUG ] 10485760 bytes (10 MB) copied
[ceph0][DEBUG ]  stderr: , 0.0110867 s, 946 MB/s
[ceph0][DEBUG ] --> Zapping successful for: /dev/sdb

从输出看，ceph-deploy调用ceph-volume lvm zap，最后是执行dd命令往disk前10M写入全0数据

创建OSD：

1
2
3
4
5
# ceph-deploy osd create ceph0 --data /dev/sdb
...
[ceph0][INFO  ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdb
...
[ceph_deploy.osd][DEBUG ] Host ceph0 is now ready for osd use.

检查OSD：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# df -h
Filesystem           Size  Used Avail Use% Mounted on
...
tmpfs                 16G   48K   16G   1% /var/lib/ceph/osd/ceph-0

# ll /var/lib/ceph/osd/ceph-0/
total 48
-rw-r--r-- 1 ceph ceph 393 Apr  2 18:11 activate.monmap
lrwxrwxrwx 1 ceph ceph  93 Apr  2 18:11 block -> /dev/ceph-5b2192ff-299c-4024-9f10-93af008e66d3/osd-block-fe7cd3b1-8513-4be5-b9a8-88fd47dca679
-rw-r--r-- 1 ceph ceph   2 Apr  2 18:11 bluefs
-rw-r--r-- 1 ceph ceph  37 Apr  2 18:11 ceph_fsid
-rw-r--r-- 1 ceph ceph  37 Apr  2 18:11 fsid
-rw------- 1 ceph ceph  55 Apr  2 18:11 keyring
-rw-r--r-- 1 ceph ceph   8 Apr  2 18:11 kv_backend
-rw-r--r-- 1 ceph ceph  21 Apr  2 18:11 magic
-rw-r--r-- 1 ceph ceph   4 Apr  2 18:11 mkfs_done
-rw-r--r-- 1 ceph ceph  41 Apr  2 18:11 osd_key
-rw-r--r-- 1 ceph ceph   6 Apr  2 18:11 ready
-rw-r--r-- 1 ceph ceph  10 Apr  2 18:11 type
-rw-r--r-- 1 ceph ceph   2 Apr  2 18:11 whoami

查看OSD block对应设备：

1
2
3
4
5
6
7
8
9
10
11
12
# lsblk /dev/sdb
NAME                                                                                                  MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sdb                                                                                                     8:16   0  3.7T  0 disk
└─ceph--5b2192ff--299c--4024--9f10--93af008e66d3-osd--block--fe7cd3b1--8513--4be5--b9a8--88fd47dca679 253:3    0  3.7T  0 lvm

# pvs /dev/sdb
  PV         VG                                        Fmt  Attr PSize PFree
  /dev/sdb   ceph-5b2192ff-299c-4024-9f10-93af008e66d3 lvm2 a--  3.64t    0

# lvs ceph-5b2192ff-299c-4024-9f10-93af008e66d3
  LV                                             VG                                        Attr       LSize Pool Origin Data%  Meta%  Move Log Cpy%Sync Convert
  osd-block-fe7cd3b1-8513-4be5-b9a8-88fd47dca679 ceph-5b2192ff-299c-4024-9f10-93af008e66d3 -wi-ao---- 3.64t

结论：

  • Ceph OSD的mount路径对应的是tmpfs，Linux基于内存的文件系统，而并没有单独的块设备与之对应
  • 整块磁盘创建一个PV，然后创建VG和一个LV给OSD的block使用

没有单独的块设备与tmpfs对应，那上面的数据存在哪里了？

答：存在OSD的metadata里了！

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
文件：osd/OSD.cc
int OSD::write_meta(CephContext *cct, ObjectStore *store, uuid_d& cluster_fsid, uuid_d& osd_fsid, int whoami)
{
    ...
    snprintf(val, sizeof(val), "%s", CEPH_OSD_ONDISK_MAGIC);
    r = store->write_meta("magic", val);
    if (r < 0)
        return r;

    snprintf(val, sizeof(val), "%d", whoami);
    r = store->write_meta("whoami", val);
    if (r < 0)
        return r;

    cluster_fsid.print(val);
    r = store->write_meta("ceph_fsid", val);
    if (r < 0)
        return r;

    string key = cct->_conf->get_val<string>("key");
    if (key.size()) {
        r = store->write_meta("osd_key", key);
        if (r < 0)
            return r;
    } else {
    ...
    r = store->write_meta("ready", "ready");
    ...    
}
    
文件：os/bluestore/BlueStore.cc
int BlueStore::write_meta(const std::string& key, const std::string& value)
{
    bluestore_bdev_label_t label;
    string p = path + "/block";
    int r = _read_bdev_label(cct, p, &label);
    if (r < 0) {
        return ObjectStore::write_meta(key, value);
    }
    label.meta[key] = value;
    r = _write_bdev_label(cct, p, label);
    assert(r == 0);
    return ObjectStore::write_meta(key, value);
}

磁盘分区

命令格式：

1
# ceph-deploy osd create [host] --data [/path/to/device-partition]

首先创建磁盘分区：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
# fdisk -l /dev/sdc
...
Disk /dev/sdc: 4000.8 GB, 4000787030016 bytes, 7814037168 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk label type: gpt
Disk identifier: 84B34A2B-5F0D-4C4F-ADCD-1974B8FD5851


#         Start          End    Size  Type            Name
 1         2048   7814037134    3.7T  Linux filesyste

# lsblk /dev/sdc
NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sdc      8:32   0  3.7T  0 disk
└─sdc1   8:33   0  3.7T  0 part

请使用gpt分区格式，其他的会报错

创建OSD：

1
2
3
4
5
6
7
8
9
10
11
# ceph-deploy osd create ceph0 --data /dev/sdc1
...
[ceph0][INFO  ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdc1
...
[ceph0][DEBUG ] --> ceph-volume lvm activate successful for osd ID: 1
[ceph0][DEBUG ] --> ceph-volume lvm activate successful for osd ID: None
[ceph0][DEBUG ] --> ceph-volume lvm create successful for: /dev/sdc1
[ceph0][INFO  ] checking OSD status...
[ceph0][DEBUG ] find the location of an executable
[ceph0][INFO  ] Running command: /bin/ceph --cluster=ceph osd stat --format=json
[ceph_deploy.osd][DEBUG ] Host ceph0 is now ready for osd use.

检查OSD：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# ll /var/lib/ceph/osd/ceph-1/
total 48
-rw-r--r-- 1 ceph ceph 393 Apr  2 18:48 activate.monmap
lrwxrwxrwx 1 ceph ceph  93 Apr  2 18:48 block -> /dev/ceph-589101eb-51c3-42b9-adad-915bfccfc4f2/osd-block-8553492d-cb56-4b69-ab9f-d0cfcf0d0970
-rw-r--r-- 1 ceph ceph   2 Apr  2 18:48 bluefs
-rw-r--r-- 1 ceph ceph  37 Apr  2 18:48 ceph_fsid
-rw-r--r-- 1 ceph ceph  37 Apr  2 18:48 fsid
-rw------- 1 ceph ceph  55 Apr  2 18:48 keyring
-rw-r--r-- 1 ceph ceph   8 Apr  2 18:48 kv_backend
-rw-r--r-- 1 ceph ceph  21 Apr  2 18:48 magic
-rw-r--r-- 1 ceph ceph   4 Apr  2 18:48 mkfs_done
-rw-r--r-- 1 ceph ceph  41 Apr  2 18:48 osd_key
-rw-r--r-- 1 ceph ceph   6 Apr  2 18:48 ready
-rw-r--r-- 1 ceph ceph  10 Apr  2 18:48 type
-rw-r--r-- 1 ceph ceph   2 Apr  2 18:48 whoam

# pvs /dev/sdc1
  PV         VG                                        Fmt  Attr PSize PFree
  /dev/sdc1  ceph-589101eb-51c3-42b9-adad-915bfccfc4f2 lvm2 a--  3.64t    0

# lvs ceph-589101eb-51c3-42b9-adad-915bfccfc4f2
  LV                                             VG                                        Attr       LSize Pool Origin Data%  Meta%  Move Log Cpy%Sync Convert
  osd-block-8553492d-cb56-4b69-ab9f-d0cfcf0d0970 ceph-589101eb-51c3-42b9-adad-915bfccfc4f2 -wi-ao---- 3.64t

结论：

  • 与使用整块磁盘基本一样，不同的只是用磁盘分区创建一个PV

逻辑卷

命令格式：

1
# ceph-deploy osd create [host] --data [vg/lv]

首先创建一个逻辑卷：

1
2
3
4
5
6
7
8
# pvcreate /dev/sdd
  Physical volume "/dev/sdd" successfully created.

# vgcreate sddvg /dev/sdd
  Volume group "sddvg" successfully created

# lvcreate -n sddlv -l 100%FREE sddvg
  Logical volume "sddlv" created.

创建OSD：

1
2
3
4
5
6
7
8
9
10
11
# ceph-deploy osd create ceph0 --data sddvg/sddlv
...
[ceph0][INFO  ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data sddvg/sddlv
...
[ceph0][DEBUG ] --> ceph-volume lvm activate successful for osd ID: 2
[ceph0][DEBUG ] --> ceph-volume lvm activate successful for osd ID: None
[ceph0][DEBUG ] --> ceph-volume lvm create successful for: sddvg/sddlv
[ceph0][INFO  ] checking OSD status...
[ceph0][DEBUG ] find the location of an executable
[ceph0][INFO  ] Running command: /bin/ceph --cluster=ceph osd stat --format=json
[ceph_deploy.osd][DEBUG ] Host ceph0 is now ready for osd use.

若指定--data /dev/sddvg/sddlv，命令会报错，会被认为是block device

检查OSD：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# ll /var/lib/ceph/osd/ceph-2/
total 48
-rw-r--r-- 1 ceph ceph 393 Apr  2 18:55 activate.monmap
lrwxrwxrwx 1 ceph ceph  16 Apr  2 18:55 block -> /dev/sddvg/sddlv
-rw-r--r-- 1 ceph ceph   2 Apr  2 18:55 bluefs
-rw-r--r-- 1 ceph ceph  37 Apr  2 18:55 ceph_fsid
-rw-r--r-- 1 ceph ceph  37 Apr  2 18:55 fsid
-rw------- 1 ceph ceph  55 Apr  2 18:55 keyring
-rw-r--r-- 1 ceph ceph   8 Apr  2 18:55 kv_backend
-rw-r--r-- 1 ceph ceph  21 Apr  2 18:55 magic
-rw-r--r-- 1 ceph ceph   4 Apr  2 18:55 mkfs_done
-rw-r--r-- 1 ceph ceph  41 Apr  2 18:55 osd_key
-rw-r--r-- 1 ceph ceph   6 Apr  2 18:55 ready
-rw-r--r-- 1 ceph ceph  10 Apr  2 18:55 type
-rw-r--r-- 1 ceph ceph   2 Apr  2 18:55 whoami

结论：

  • 与前两个一致，区别只是自己收到创建了PV，VG，LV

指定block.wal和block.db设备创建OSD

当指定block.wal或block.db时，对应的设备可以为两种：

  1. 物理磁盘，但必须是一个磁盘分区
  2. 逻辑卷（a logical volume of LVM）

这里只区分指定的block.wal或block.db设备，data设备选择整块磁盘。

bluestore的block.db和block.wal大小

默认值如下，db size = 0，wal size = 100663296，都比较小。

1
2
3
4
5
6
7
8
9
10
11
# ceph-conf --show-config | grep bluestore_block
bluestore_block_create = true
bluestore_block_db_create = false
bluestore_block_db_path =
bluestore_block_db_size = 0
bluestore_block_path =
bluestore_block_preallocate_file = false
bluestore_block_size = 10737418240
bluestore_block_wal_create = false
bluestore_block_wal_path =
bluestore_block_wal_size = 100663296

参考：https://marc.info/?l=ceph-devel&m=149978799900866&w=2

对block.db和block.wal的大小要求也都比较小，后续测试我们选择block.db和block.wal为10G。

磁盘分区

命令格式：

1
# ceph-deploy osd create [host] --data [/path/to/device] --block-db [/path/to/device-partition] --block-wal [/path/to/device-partition]

首先创建两个磁盘分区给block.wal和block.db使用：

1
2
3
4
5
6
7
8
9
10
# parted -s /dev/sdf print
Model: ATA INTEL SSDSC2BB48 (scsi)
Disk /dev/sdf: 480GB
Sector size (logical/physical): 512B/4096B
Partition Table: gpt
Disk Flags:

Number  Start   End     Size    File system  Name  Flags
 1      1049kB  10.7GB  10.7GB
 2      10.7GB  21.5GB  10.7GB

创建OSD：

1
2
3
4
5
6
7
8
9
10
11
12
# ceph-deploy disk zap ceph0 /dev/sde
# ceph-deploy osd create ceph0 --data /dev/sde --block-db /dev/sdf1 --block-wal /dev/sdf2
...
[ceph0][INFO  ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sde --block.wal /dev/sdf2 --block.db /dev/sdf1
...
[ceph0][DEBUG ] --> ceph-volume lvm activate successful for osd ID: 3
[ceph0][DEBUG ] --> ceph-volume lvm activate successful for osd ID: None
[ceph0][DEBUG ] --> ceph-volume lvm create successful for: /dev/sde
[ceph0][INFO  ] checking OSD status...
[ceph0][DEBUG ] find the location of an executable
[ceph0][INFO  ] Running command: /bin/ceph --cluster=ceph osd stat --format=json
[ceph_deploy.osd][DEBUG ] Host ceph0 is now ready for osd use.

检查OSD：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# df -h
Filesystem           Size  Used Avail Use% Mounted on
...
tmpfs                 16G   56K   16G   1% /var/lib/ceph/osd/ceph-3

# ll /var/lib/ceph/osd/ceph-3
total 56
-rw-r--r-- 1 ceph ceph 393 Apr  3 09:20 activate.monmap
lrwxrwxrwx 1 ceph ceph  93 Apr  3 09:20 block -> /dev/ceph-7777d5e4-9b81-4c17-916d-7a1e48f6268e/osd-block-3ed97688-03b4-4ca6-a497-bbd30e865852
lrwxrwxrwx 1 root root   9 Apr  3 09:20 block.db -> /dev/sdf1
lrwxrwxrwx 1 root root   9 Apr  3 09:20 block.wal -> /dev/sdf2
-rw-r--r-- 1 ceph ceph   2 Apr  3 09:20 bluefs
-rw-r--r-- 1 ceph ceph  37 Apr  3 09:20 ceph_fsid
-rw-r--r-- 1 ceph ceph  37 Apr  3 09:20 fsid
-rw------- 1 ceph ceph  55 Apr  3 09:20 keyring
-rw-r--r-- 1 ceph ceph   8 Apr  3 09:20 kv_backend
-rw-r--r-- 1 ceph ceph  21 Apr  3 09:20 magic
-rw-r--r-- 1 ceph ceph   4 Apr  3 09:20 mkfs_done
-rw-r--r-- 1 ceph ceph  41 Apr  3 09:20 osd_key
-rw-r--r-- 1 ceph ceph  10 Apr  3 09:20 path_block.db
-rw-r--r-- 1 ceph ceph  10 Apr  3 09:20 path_block.wal
-rw-r--r-- 1 ceph ceph   6 Apr  3 09:20 ready
-rw-r--r-- 1 ceph ceph  10 Apr  3 09:20 type
-rw-r--r-- 1 ceph ceph   2 Apr  3 09:20 whoami

结论：

  • 与使用单块盘基本一样，不同的只是指定了block.db -> /dev/sdf1和block.wal -> /dev/sdf2

逻辑卷

命令格式：

1
# ceph-deploy osd create [host] --data [/path/to/device] --block-db [vg/lv]  --block-wal [vg/lv]

首先创建两个逻辑卷给block.wal和block.db使用：

1
2
3
4
# pvcreate /dev/sdm
# vgcreate ssdvg /dev/sdm
# lvcreate -n db-lv-0 -L 4G ssdvg
# lvcreate -n wal-lv-0 -L 8G ssdvg

创建OSD：

1
2
3
4
5
6
7
8
9
10
11
12
# ceph-deploy disk zap ceph0 /dev/sdg
# ceph-deploy osd create ceph0 --data /dev/sdg --block-db ssdvg/db-lv-0 --block-wal ssdvg/wal-lv-0
...
[ceph0][INFO  ] Running command: /usr/sbin/ceph-volume --cluster ceph lvm create --bluestore --data /dev/sdg --block.wal ssdvg/wal-lv-0 --block.db ssdvg/db-lv-0
...
[ceph0][DEBUG ] --> ceph-volume lvm activate successful for osd ID: 4
[ceph0][DEBUG ] --> ceph-volume lvm activate successful for osd ID: None
[ceph0][DEBUG ] --> ceph-volume lvm create successful for: /dev/sdg
[ceph0][INFO  ] checking OSD status...
[ceph0][DEBUG ] find the location of an executable
[ceph0][INFO  ] Running command: /bin/ceph --cluster=ceph osd stat --format=json
[ceph_deploy.osd][DEBUG ] Host ceph0 is now ready for osd use.

若指定--block-db /dev/ssdvg/db-lv-0 --block-wal /dev/ssdvg/wal-lv-0，命令会报错

检查OSD：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# df -h
Filesystem           Size  Used Avail Use% Mounted on
...
tmpfs                 16G   56K   16G   1% /var/lib/ceph/osd/ceph-4

# ll /var/lib/ceph/osd/ceph-4
total 56
-rw-r--r-- 1 ceph ceph 393 Apr  3 09:30 activate.monmap
lrwxrwxrwx 1 ceph ceph  93 Apr  3 09:30 block -> /dev/ceph-017b646a-0332-4677-967b-95033a3a33ab/osd-block-24228535-1fb3-4bcd-bb87-f6d5f49ed24d
lrwxrwxrwx 1 root root  18 Apr  3 09:30 block.db -> /dev/ssdvg/db-lv-0
lrwxrwxrwx 1 root root  19 Apr  3 09:30 block.wal -> /dev/ssdvg/wal-lv-0
-rw-r--r-- 1 ceph ceph   2 Apr  3 09:30 bluefs
-rw-r--r-- 1 ceph ceph  37 Apr  3 09:30 ceph_fsid
-rw-r--r-- 1 ceph ceph  37 Apr  3 09:30 fsid
-rw------- 1 ceph ceph  55 Apr  3 09:30 keyring
-rw-r--r-- 1 ceph ceph   8 Apr  3 09:30 kv_backend
-rw-r--r-- 1 ceph ceph  21 Apr  3 09:30 magic
-rw-r--r-- 1 ceph ceph   4 Apr  3 09:30 mkfs_done
-rw-r--r-- 1 ceph ceph  41 Apr  3 09:30 osd_key
-rw-r--r-- 1 ceph ceph  19 Apr  3 09:30 path_block.db
-rw-r--r-- 1 ceph ceph  20 Apr  3 09:30 path_block.wal
-rw-r--r-- 1 ceph ceph   6 Apr  3 09:30 ready
-rw-r--r-- 1 ceph ceph  10 Apr  3 09:30 type
-rw-r--r-- 1 ceph ceph   2 Apr  3 09:30 whoami

结论：

  • 与使用单块盘基本一样，不同的只是指定了block.db -> /dev/ssdvg/db-lv-0和block.wal -> /dev/ssdvg/wal-lv-0

Posted on | In |

概述

在前面的文章中介绍了 SSD加速SATA盘之flashcache策略。
一般我们也推荐选择稳定的flashcache策略来做SSD加速SATA盘，但在实践中，发现其在CentOS上编译安装还是很麻烦的，这里就抓紧研究实践了下bcache策略。

另外bcache使用可以用一块SSD来缓存多块SATA盘，对于使用中随时变动磁盘的应用场景来说，操作非常便捷。

并且官网说bcache的性能完全优于flashcache，参考：

http://www.accelcloud.com/2012/04/18/linux-flashcache-and-bcache-performance-testing/

Bcache

介绍：

https://wiki.archlinux.org/index.php/Bcache

https://bcache.evilpiepirate.org/

Bcache在Linux kernel 3.10版本加入了mainline，使用它只需要主机的kernel版本大于3.10即可。

bcache-tools 源码：https://evilpiepirate.org/git/bcache-tools.git

Ubuntu上安装

系统信息

1
2
3
4
5
6
7
8
# lsb_release -a
No LSB modules are available.
Distributor ID:    Ubuntu
Description:    Ubuntu 16.04.2 LTS
Release:    16.04
Codename:    xenial
# uname -r
4.4.0-72-generic

编译安装

加载系统的bcache module：

1
2
3
# lsmod | grep bcache
# modprobe bcache
# lsmod | grep bcache

编译安装bcace-tools：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
# apt-get install -y pkg-config libblkid-dev
# git clone https://evilpiepirate.org/git/bcache-tools.git
# cd bcache-tools
# make
cc -O2 -Wall -g `pkg-config --cflags uuid blkid`   -c -o bcache.o bcache.c
bcache.c:125:9: warning: ‘crc_table’ is static but used in inline function ‘crc64’ which is not static
   crc = crc_table[i] ^ (crc << 8);
         ^
cc -O2 -Wall -g `pkg-config --cflags uuid blkid`    make-bcache.c bcache.o  `pkg-config --libs uuid blkid` -o make-bcache
/tmp/ccW6rXtD.o: In function `write_sb':
/root/yangguanjun/bcache-tools/make-bcache.c:277: undefined reference to `crc64'
collect2: error: ld returned 1 exit status
<builtin>: recipe for target 'make-bcache' failed
make: *** [make-bcache] Error 1

网上搜索有这个bug的fix，如下：

https://www.spinics.net/lists/linux-bcache/msg02847.html

1
2
3
4
5
6
7
8
9
10
11
12
--- a/bcache.c
+++ b/bcache.c

@@ -115,7 +115,7 @@ static const uint64_t crc_table[256] = {
       0x9AFCE626CE85B507ULL
};

-inline uint64_t crc64(const void *_data, size_t len)
+uint64_t crc64(const void *_data, size_t len)
{
       uint64_t crc = 0xFFFFFFFFFFFFFFFFULL;
       const unsigned char *data = _data;

按上面patch修改bcache.c后，编译安装正常。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# make
cc -O2 -Wall -g `pkg-config --cflags uuid blkid`   -c -o bcache.o bcache.c
cc -O2 -Wall -g `pkg-config --cflags uuid blkid`    make-bcache.c bcache.o  `pkg-config --libs uuid blkid` -o make-bcache
cc -O2 -Wall -g `pkg-config --cflags uuid blkid`    probe-bcache.c  `pkg-config --libs uuid blkid` -o probe-bcache
cc -O2 -Wall -g -std=gnu99    bcache-super-show.c bcache.o  `pkg-config --libs uuid` -o bcache-super-show
cc -O2 -Wall -g   -c -o bcache-register.o bcache-register.c
cc   bcache-register.o   -o bcache-register
# make install
install -m0755 make-bcache bcache-super-show    /usr/sbin/
install -m0755 probe-bcache bcache-register        /lib/udev/
install -m0644 69-bcache.rules    /lib/udev/rules.d/
install -m0644 -- *.8 /usr/share/man/man8/
install -D -m0755 initramfs/hook    /usr/share/initramfs-tools/hooks/bcache
install -D -m0755 initcpio/install    /usr/lib/initcpio/install/bcache
install -D -m0755 dracut/module-setup.sh /lib/dracut/modules.d/90bcache/module-setup.sh

CentOS上安装

因为bcache在kernel 3.10版本才进入主线，所以我们要保证CentOS的内核版本大于3.10，所以CenOS 6就不要尝试了，直接用新的CentOS 7吧。

系统信息

1
2
3
4
5
6
7
8
# lsb_release -a
LSB Version:    :core-4.1-amd64:core-4.1-noarch
Distributor ID:    CentOS
Description:    CentOS Linux release 7.3.1611 (Core)
Release:    7.3.1611
Codename:    Core
# uname -r
3.10.0-693.17.1.el7.x86_64

编译安装

加载系统的bcache模块：

1
2
3
# lsmod | grep bcache
# modprobe bcache
modprobe: FATAL: Module bcache not found.

查看发现在kernel 3.10.0-693 版本里，默认是不编译bcache模块的，参考：

https://lakelight.net/2017/12/20/bcache-centos-7.html

这里就需要下载当前内核版本的源码，重新编译内核bcache模块，然后再加载bcache模块。

鉴于之前CentOS上安装flashcache时探索了内核版本的升级，所以这里在已经升级内核版本的机器上尝试，发现kernel 4.4版本默认已经编译好了bcache模块，所以在CentOS上使用cache时，还是建议升级到4.4版本内核。

1
2
3
4
5
6
7
# uname -r
4.4.115-1.el7.elrepo.x86_64

# lsmod | grep bcache
# modprobe bcache
# lsmod | grep bcache
bcache                233472  0

编译安装bcace-tools：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# yum install -y git pkgconfig libblkid-devel
# git clone https://evilpiepirate.org/git/bcache-tools.git
# cd bcache-tools/
# make
cc -O2 -Wall -g `pkg-config --cflags uuid blkid`   -c -o bcache.o bcache.c
cc -O2 -Wall -g `pkg-config --cflags uuid blkid`    make-bcache.c bcache.o  `pkg-config --libs uuid blkid` -o make-bcache
cc -O2 -Wall -g `pkg-config --cflags uuid blkid`    probe-bcache.c  `pkg-config --libs uuid blkid` -o probe-bcache
cc -O2 -Wall -g -std=gnu99    bcache-super-show.c bcache.o  `pkg-config --libs uuid` -o bcache-super-show
cc -O2 -Wall -g   -c -o bcache-register.o bcache-register.c
cc   bcache-register.o   -o bcache-register
# make install
install -m0755 make-bcache bcache-super-show    /usr/sbin/
install -m0755 probe-bcache bcache-register        /lib/udev/
install -m0644 69-bcache.rules    /lib/udev/rules.d/
install -m0644 -- *.8 /usr/share/man/man8/
install -D -m0755 initramfs/hook    /usr/share/initramfs-tools/hooks/bcache
install -D -m0755 initcpio/install    /usr/lib/initcpio/install/bcache
install -D -m0755 dracut/module-setup.sh /lib/dracut/modules.d/90bcache/module-setup.sh

Bcache使用

下面在CentOS机器上，介绍如何使用bcache。

硬盘信息

1
2
3
4
5
# fdisk -l | grep dev
...
Disk /dev/vdb: 107.4 GB, 107374182400 bytes, 209715200 sectors
Disk /dev/vdc: 107.4 GB, 107374182400 bytes, 209715200 sectors
Disk /dev/vdd: 53.7 GB, 53687091200 bytes, 104857600 sectors

这里使用三块盘：vdb、vdc、vdd。

其中vdb、vdc是容量型磁盘，vdd是性能型磁盘，实验用vdd通过bcache加速vdb和vdc。

使用步骤

与bcache相关的命令有：make-bcache和bcache-super-show

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# make-bcache
Please supply a device
Usage: make-bcache [options] device
	-C, --cache		Format a cache device
	-B, --bdev		Format a backing device
	-b, --bucket		bucket size
	-w, --block		block size (hard sector size of SSD, often 2k)
	-o, --data-offset	data offset in sectors
	    --cset-uuid		UUID for the cache set
	    --writeback		enable writeback
	    --discard		enable discards
	    --cache_replacement_policy=(lru|fifo)
	-h, --help		display this help and exit
	
# bcache-super-show
Usage: bcache-super-show [-f] <device>

创建backing device

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# make-bcache -B /dev/vdb
UUID:            c602abab-bf5a-4b51-b7f6-1492d34239f4
Set UUID:        423e1910-f61a-45fa-8cdf-a23aca3b5eb8
version:        1
block_size:        1
data_offset:        16

# bcache-super-show /dev/vdb
sb.magic		ok
sb.first_sector		8 [match]
sb.csum			1376BA45B5F924B [match]
sb.version		1 [backing device]

dev.label		(empty)
dev.uuid		c602abab-bf5a-4b51-b7f6-1492d34239f4
dev.sectors_per_block	1
dev.sectors_per_bucket	1024
dev.data.first_sector	16
dev.data.cache_mode	0 [writethrough]
dev.data.cache_state	1 [clean]

cset.uuid		4b60c663-7720-4dea-a17a-e9316078e796

创建cache device

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
# make-bcache -C /dev/vdd
UUID:            050998ce-403c-45d7-a89b-492379644c1b
Set UUID:        4b60c663-7720-4dea-a17a-e9316078e796
version:        0
nbuckets:        102400
block_size:        1
bucket_size:        1024
nr_in_set:        1
nr_this_dev:        0
first_bucket:        1

# bcache-super-show /dev/vdd
sb.magic        ok
sb.first_sector        8 [match]
sb.csum            68CDDDC337A2E296 [match]
sb.version        3 [cache device]

dev.label        (empty)
dev.uuid        050998ce-403c-45d7-a89b-492379644c1b
dev.sectors_per_block    1
dev.sectors_per_bucket    1024
dev.cache.first_sector    1024
dev.cache.cache_sectors    104856576
dev.cache.total_sectors    104857600
dev.cache.ordered    yes
dev.cache.discard    no
dev.cache.pos        0
dev.cache.replacement    0 [lru]

cset.uuid        4b60c663-7720-4dea-a17a-e9316078e796

绑定backing device到cache device

1
2
3
4
5
6
7
8
9
# echo "4b60c663-7720-4dea-a17a-e9316078e796" > /sys/block/bcache0/bcache/attach
# lsblk
NAME      MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
...
vdb       253:16   0  100G  0 disk
└─bcache0 251:0    0  100G  0 disk
vdc       253:32   0  100G  0 disk
vdd       253:48   0   50G  0 disk
└─bcache0 251:0    0  100G  0 disk

查看bcache相关信息

1、state

1
2
# cat /sys/block/bcache0/bcache/state
clean

state的几个状态：

  • no cache：该backing device没有attach任何caching device
  • clean：一切正常，缓存是干净的
  • dirty：一切正常，已启用回写，缓存是脏的
  • inconsistent：遇到问题，后台设备与缓存设备不同步

2、缓存数据量

1
2
# cat /sys/block/bcache0/bcache/dirty_data
0.0k

3、缓存模式

1
2
3
4
5
# cat /sys/block/bcache0/bcache/cache_mode
[writethrough] writeback writearound none
# echo writearound > /sys/block/bcache0/bcache/cache_mode
# cat /sys/block/bcache0/bcache/cache_mode
writethrough writeback [writearound] none

4、writeback信息

1
2
3
# cat /sys/block/bcache0/bcache/writeback_
writeback_delay                writeback_percent              writeback_rate_debug           writeback_rate_p_term_inverse  writeback_running
writeback_metadata             writeback_rate                 writeback_rate_d_term          writeback_rate_update_seconds

解绑backing device的cache device

1
2
3
4
5
6
7
8
9
10
11
# echo "697b764f-b3ef-4675-8761-d9518a12089c" > /sys/block/bcache0/bcache/detach
# lsblk
NAME      MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
...
vdb       253:16   0  100G  0 disk
└─bcache0 251:0    0  100G  0 disk
vdc       253:32   0  100G  0 disk
vdd       253:48   0   50G  0 disk

# cat /sys/block/vdb/bcache/state
no cache

解绑后设备可以继续使用，只是没有cache device的加速

添加新backing device

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
# make-bcache -B /dev/vdc
UUID:            cc790e62-b3eb-4237-8265-dd1b619e15c0
Set UUID:        edb2b1d0-9eeb-4a8a-b811-3dafd676fac0
version:        1
block_size:        1
data_offset:        16

# echo "4b60c663-7720-4dea-a17a-e9316078e796" > /sys/block/bcache1/bcache/attach
# lsblk
NAME      MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
...
vdb       253:16   0  100G  0 disk
└─bcache0 251:0    0  100G  0 disk
vdc       253:32   0  100G  0 disk
└─bcache1 251:1    0  100G  0 disk
vdd       253:48   0   50G  0 disk
└─bcache1 251:1    0  100G  0 disk

使用bcache device

1
2
3
4
5
6
7
8
# mkfs.ext4 /dev/bcache1
# mount /dev/bcache1 /mnt/
# df -h
Filesystem      Size  Used Avail Use% Mounted on
...
/dev/bcache1     99G   61M   94G   1% /mnt

# umount /mnt/

注销bcache device

1
2
3
4
5
6
7
8
9
# echo 1 > /sys/block/vdc/bcache/stop
# echo 1 > /sys/block/vdb/bcache/stop
# echo 1 > /sys/fs/bcache/10057a1c-15a2-4631-a6d2-f4652d37645d/unregister
# lsblk
NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
...
vdb    253:16   0  100G  0 disk
vdc    253:32   0  100G  0 disk
vdd    253:48   0   50G  0 disk

echo的数字不重要，可为任何值 ;)

快捷创建bcache device

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
# make-bcache -B /dev/vdb /dev/vdc -C /dev/vdd
UUID:			09f971eb-6063-4f94-bdac-d7d7117c0e0f
Set UUID:		697b764f-b3ef-4675-8761-d9518a12089c
version:		0
nbuckets:		102400
block_size:		1
bucket_size:		1024
nr_in_set:		1
nr_this_dev:		0
first_bucket:		1
UUID:			b45301fa-8932-4194-9518-ab681f37d9c9
Set UUID:		697b764f-b3ef-4675-8761-d9518a12089c
version:		1
block_size:		1
data_offset:		16
UUID:			2c0452f7-76de-4319-bd3c-2a73b4fa4b68
Set UUID:		697b764f-b3ef-4675-8761-d9518a12089c
version:		1
block_size:		1
data_offset:		16
[root@lvm-centos-tst ~]# lsblk
NAME      MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
...
vdb       253:16   0  100G  0 disk
└─bcache0 251:0    0  100G  0 disk
vdc       253:32   0  100G  0 disk
└─bcache1 251:1    0  100G  0 disk
vdd       253:48   0   50G  0 disk
├─bcache0 251:0    0  100G  0 disk
└─bcache1 251:1    0  100G  0 disk

遇到的问题

make-bcache命令有提示

之前做过bcache的device，重做bcache有提示

1
2
3
4
5
6
7
8
# make-bcache -B /dev/vdb
Already a bcache device on /dev/vdb, overwrite with --wipe-bcache

# lsblk
NAME      MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
...
vdb       253:16   0  100G  0 disk
└─bcache0 251:0    0  100G  0 disk

虽说有提示，但实际bcache device已经创建成果

针对上述情况，可以通过写device前一部分数据的方法解决

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
可以通过dd命令来清理device的前部分数据：
# dd if=/dev/zero of=/dev/vdb bs=1M count=100 oflag=direct
# dd if=/dev/zero of=/dev/vdd bs=1M count=100 oflag=direct

再创建bcache device，就不会报错了：
# make-bcache -B /dev/vdb
UUID:            c602abab-bf5a-4b51-b7f6-1492d34239f4
Set UUID:        423e1910-f61a-45fa-8cdf-a23aca3b5eb8
version:        1
block_size:        1
data_offset:        16

# make-bcache -C /dev/vdd
UUID:            050998ce-403c-45d7-a89b-492379644c1b
Set UUID:        4b60c663-7720-4dea-a17a-e9316078e796
version:        0
nbuckets:        102400
block_size:        1
bucket_size:        1024
nr_in_set:        1
nr_this_dev:        0
first_bucket:        1

设备没umount就直接注销

没有umount，注销bcache device后，设备依旧可以使用，umount后设备消失

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
# lsblk
NAME      MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
...
vdb       253:16   0  100G  0 disk
└─bcache0 251:0    0  100G  0 disk /mnt
vdc       253:32   0  100G  0 disk
vdd       253:48   0   50G  0 disk
└─bcache0 251:0    0  100G  0 disk /mnt

# mount | grep bcache
/dev/bcache0 on /mnt type ext4 (rw,relatime,seclabel,data=ordered)

# echo 1 > /sys/fs/bcache/4b60c663-7720-4dea-a17a-e9316078e796/unregister
# echo 0 > /sys/block/vdb/bcache/stop

# lsblk
NAME      MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
...
vdb       253:16   0  100G  0 disk
└─bcache0 251:0    0  100G  0 disk /mnt
vdc       253:32   0  100G  0 disk
vdd       253:48   0   50G  0 disk

# ls /sys/block/vdb/bcache
ls: cannot access /sys/block/vdb/bcache: No such file or directory

# cd /mnt/
# touch tstfile
# umount  /mnt/

# lsblk
NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
...
vdb    253:16   0  100G  0 disk
vdc    253:32   0  100G  0 disk
vdd    253:48   0   50G  0 disk

Posted on | In |

概述

通常SATA盘的性能比较低，对于大多数应用来说性能不够，但纯SSD的盘又比较昂贵，结合这两种盘的使用策略是业内讨论的一个热点，也有很多成熟的方案来使用。

之前有文章讨论过cache策略，我们这里选择通用的flashcache方案来用SSD盘加速SATA盘。

flashcache

源码：https://github.com/facebookarchive/flashcache
参考：http://www.yangguanjun.com/2018/01/30/lvm-with-cache/

Ubuntu上安装

测试使用的是”Ubuntu 16.04.2 LTS”，它的内核比较新，所以这里编译安装flashcache很方便。

系统信息

1
2
3
4
5
6
7
8
# lsb_release -a
No LSB modules are available.
Distributor ID:	Ubuntu
Description:	Ubuntu 16.04.2 LTS
Release:	16.04
Codename:	xenial
# uname -r
4.4.0-62-generic

编译安装

1
2
3
4
5
6
7
8
9
10
11
12
13
# apt install -y git make gcc
# git clone git@github.com:facebookarchive/flashcache.git
# cd flashcache/
# make
# make install
# modprobe flashcache

# cat /proc/flashcache/flashcache_version
Flashcache Version : flashcache-3.1.1
git commit: 1.0-248-g437afbfe233e

# flashcache_
flashcache_create    flashcache_destroy   flashcache_load      flashcache_setioctl

CentOS上安装

测试使用的是”CentOS Linux release 7.3.1611”，它的内核比较旧，在安装flashcache时遇到很多问题。

系统信息

1
2
3
4
5
6
7
8
9
# lsb_release -a
...
Distributor ID:    CentOS
Description:    CentOS Linux release 7.3.1611 (Core)
Release:    7.3.1611
Codename:    Core

# uname -a
Linux xs732 3.10.0-514.el7.x86_64 #1 SMP Tue Nov 22 16:42:41 UTC 2016 x86_64 x86_64 x86_64 GNU/Linux

编译安装

1
2
3
4
5
6
7
8
# yum install -y git make gcc 
# git clone git@github.com:facebookarchive/flashcache.git
# cd flashcache
# make
...
/root/yangguanjun/flashcache/src/flashcache_conf.c:1230:11: error: too many arguments to function ‘wait_on_bit_lock’
           flashcache_wait_schedule, TASK_UNINTERRUPTIBLE);
...

网上可搜索到该问题：https://github.com/facebookarchive/flashcache/issues/191

升级内核

尝试升级CentOS的内核版本，但是编译flashcache还是一样的错误。

1
2
3
# yum update
# uname -a
Linux server0 3.10.0-693.17.1.el7.x86_64 #1 SMP Thu Jan 25 20:13:58 UTC 2018 x86_64 x86_64 x86_64 GNU/Linux

无奈，选择升级CentOS内核到最新版本4.15。

参考：http://www.jiagoumi.com/work/1167.html

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
先导入elrepo的key，然后安装elrepo的yum源：
# rpm -import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
# rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm

列出可用的内核相关包：
# yum --disablerepo="*" --enablerepo="elrepo-kernel" list available
…
kernel-lt.x86_64               4.4.115-1.el7.elrepo                  elrepo-kernel
kernel-lt-devel.x86_64         4.4.115-1.el7.elrepo                  elrepo-kernel
...
kernel-ml.x86_64               4.15.1-1.el7.elrepo                   elrepo-kernel
kernel-ml-devel.x86_64         4.15.1-1.el7.elrepo                   elrepo-kernel
...

安装最新的主线稳定内核：
# yum -y --enablerepo=elrepo-kernel install kernel-ml.x86_64 kernel-ml-devel.x86_64

查看系统kernel默认启动顺序：
# awk -F\' '$1=="menuentry " {print $2}' /etc/grub2.cfg
CentOS Linux (4.15.1-1.el7.elrepo.x86_64) 7 (Core)
CentOS Linux (3.10.0-514.el7.x86_64) 7 (Core)
CentOS Linux (0-rescue-836c8e52784b401db086b015b56e2fba) 7 (Core)

修改系统kernel默认启动内核：
# vim /etc/default/grub
...
GRUB_DEFAULT=saved     修改为 0
...

重新创建内核配置：
# grub2-mkconfig -o /boot/grub2/grub.cfg

重启机器：
# reboot -nf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# uname -r
4.15.1-1.el7.elrepo.x86_64

# make
make -C src KERNEL_TREE=/usr/src/kernels/4.15.1-1.el7.elrepo.x86_64 PWD=/root/flashcache/src all
...
/root/flashcache/src/flashcache_main.c: In function ‘dm_io_async_bvec_pl’:
/root/flashcache/src/flashcache_main.c:119:6: error: ‘struct dm_io_request’ has no member named ‘bi_rw’
  iorq.bi_rw = rw;
      ^
/root/flashcache/src/flashcache_main.c: In function ‘dm_io_async_bvec’:
/root/flashcache/src/flashcache_main.c:143:6: error: ‘struct dm_io_request’ has no member named ‘bi_rw’
  iorq.bi_rw = rw;
...

很无语，最新kernel的4.15版本竟然也编译不过flashcache。

查看之前编译安装flashcache成功的Ubuntu平台上的内核版本为：4.4.0-62-generic，最后决定尝试 4.4的kernel版本，步骤与上面的步骤类似，差别如下：

1
2
3
4
5
6
7
重新升级系统内核，选择kernel-lt*
# yum -y --enablerepo=elrepo-kernel install kernel-lt.x86_64 kernel-lt-devel.x86_64

之后重启机器

# uname -r
4.4.115-1.el7.elrepo.x86_64

再尝试编译安装flashcache，一切正常O(∩_∩)O哈哈~

1
2
3
4
5
# cd flashcache
# make
# make install
# flashcache_
flashcache_create    flashcache_destroy   flashcache_load      flashcache_setioctl

flashcache使用

硬盘信息

1
2
3
4
5
# fdisk -l | grep dev
...
Disk /dev/vdb: 107.4 GB, 107374182400 bytes, 209715200 sectors
Disk /dev/vdc: 107.4 GB, 107374182400 bytes, 209715200 sectors
Disk /dev/vdd: 53.7 GB, 53687091200 bytes, 104857600 sectors

这里使用三块盘：vdb、vdc、vdd。

其中vdb、vdc是容量型磁盘，vdd是性能型磁盘，实验用vdd通过bcache加速vdb和vdc。

使用步骤

与flashcache相关的命令好几个，如下：

1
2
3
4
5
6
7
8
9
10
11
12
13
# flashcache_create
Usage: flashcache_create [-v] [-p back|thru|around] [-w] [-b block size] [-m md block size] [-s cache size] [-a associativity] cachedev ssd_devname disk_devname
Usage : flashcache_create Cache Mode back|thru|around is required argument
Usage : flashcache_create Default units for -b, -m, -s are sectors, or specify in k/M/G. Default associativity is 512.

# flashcache_destroy
Usage: flashcache_destroy ssd_devname

# flashcache_load
Usage: flashcache_load ssd_devname [cachedev]

# flashcache_setioctl
Usage: flashcache_setioctl (-c | -a | -r) (-b pid |-w pid) ssd_devname

另外flashcache是通过device mapper来做的设备映射和缓存，所以对flashcache device的操作命令也有dmsetup，使用如下：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
# dmsetup
Usage:

dmsetup
        [--version] [-h|--help [-c|-C|--columns]]
        [-v|--verbose [-v|--verbose ...]] [-f|--force]
        [--checks] [--manglename {none|hex|auto}]
        [-r|--readonly] [--noopencount] [--noflush] [--nolockfs] [--inactive]
        [--udevcookie <cookie>] [--noudevrules] [--noudevsync] [--verifyudev]
        [-y|--yes] [--readahead {[+]<sectors>|auto|none}] [--retry]
        [-c|-C|--columns] [-o <fields>] [-O|--sort <sort_fields>]
        [-S|--select <selection>] [--nameprefixes] [--noheadings]
        [--separator <separator>]

	help [-c|-C|--columns]
	create <dev_name>
	    [-j|--major <major> -m|--minor <minor>]
	    [-U|--uid <uid>] [-G|--gid <gid>] [-M|--mode <octal_mode>]
	    [-u|uuid <uuid>] [--addnodeonresume|--addnodeoncreate]
	    [--readahead {[+]<sectors>|auto|none}]
	    [-n|--notable|--table {<table>|<table_file>}]
	remove [--deferred] [-f|--force] [--retry] <device>
	remove_all [-f|--force]
	suspend [--noflush] [--nolockfs] <device>
	resume [--noflush] [--nolockfs] <device>
	       [--addnodeonresume|--addnodeoncreate]
	       [--readahead {[+]<sectors>|auto|none}]
	load <device> [<table>|<table_file>]
	clear <device>
	reload <device> [<table>|<table_file>]
	wipe_table [-f|--force] [--noflush] [--nolockfs] <device>
	rename <device> [--setuuid] <new_name_or_uuid>
	message <device> <sector> <message>
	ls [--target <target_type>] [--exec <command>] [-o <options>] [--tree]
	info [<device>]
	deps [-o <options>] [<device>]
	stats <command> [<options>] [<devices>]
	status [<device>] [--noflush] [--target <target_type>]
	table [<device>] [--target <target_type>] [--showkeys]
	wait <device> [<event_nr>] [--noflush]
	mknodes [<device>]
	mangle [<device>]
	udevcreatecookie
	udevreleasecookie [<cookie>]
	udevflags <cookie>
	udevcomplete <cookie>
	udevcomplete_all [<age_in_minutes>]
	udevcookies
	targets
	version
	setgeometry <device> <cyl> <head> <sect> <start>
	splitname <device> [<subsystem>]

<device> may be device name or -u <uuid> or -j <major> -m <minor>
<mangling_mode> is one of 'none', 'auto' and 'hex'.
<fields> are comma-separated.  Use 'help -c' for list.
Table_file contents may be supplied on stdin.
Options are: devno, devname, blkdevname.
Tree specific options are: ascii, utf, vt100; compact, inverted, notrunc;
                           blkdevname, [no]device, active, open, rw and uuid.

磁盘规划

因为flashcache要求一个ssd_device对应一个sata_device，所以在使用之前，我们要规划好哪些SSD盘做缓存？哪些SATA盘需要做flashcache？以便给出合理规划。

这里我们有一个SSD盘，两个SATA盘，所以把SSD盘分为两个分区。

1
2
3
4
5
6
7
8
9
10
11
12
13
# fdisk /dev/vdd
...
# fdisk -l /dev/vdd
Disk /dev/vdd: 53.7 GB, 53687091200 bytes, 104857600 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk label type: dos
Disk identifier: 0xa8f11a03

   Device Boot      Start         End      Blocks   Id  System
/dev/vdd1            2048    52430847    26214400   83  Linux
/dev/vdd2        52430848   104857599    26213376   83  Linux

缓存模式

flashcache支持三种缓存模式：

  • Writeback : 对于写入，首先会写入到Cache中，同时将对于block的元数据dirty bit，但是并不会立即写入后备的device
  • Writethrough : 对于写入，写入到Cache中，同时也会将数据写入backing device，知道写完backing device，才算写完
  • Writearound : 写入的时候，绕过Cache，直接写入backing device，即SSD只当读缓存

三种缓存模式的区别如下图：

flashcache-cachemodes

创建flashcache device

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
后端设备为一分区：
# flashcache_create -p back fcache-dev1 /dev/vdd1 /dev/vdb1
cachedev fcache-dev1, ssd_devname /dev/vdd1, disk_devname /dev/vdb1 cache mode WRITE_BACK
block_size 8, md_block_size 8, cache_size 0
Flashcache metadata will use 137MB of your 3951MB main memory

# lsblk
NAME            MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
...
vdb             253:16   0  100G  0 disk
└─vdb1          253:17   0  100G  0 part
  └─fcache-dev1 252:0    0  100G  0 dm
vdd             253:48   0   50G  0 disk
├─vdd1          253:49   0   25G  0 part
│ └─fcache-dev1 252:0    0  100G  0 dm
└─vdd2          253:50   0   25G  0 part

后端设备为整块磁盘：
# flashcache_create -p back fcache-dev2 /dev/vdd2 /dev/vdc
cachedev fcache-dev2, ssd_devname /dev/vdd2, disk_devname /dev/vdc cache mode WRITE_BACK
block_size 8, md_block_size 8, cache_size 0
Flashcache metadata will use 137MB of your 3951MB main memory

# lsblk
NAME            MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
...
vdb             253:16   0  100G  0 disk
└─vdb1          253:17   0  100G  0 part
  └─fcache-dev1 252:0    0  100G  0 dm
vdc             253:32   0  100G  0 disk
└─fcache-dev2   252:1    0  100G  0 dm
vdd             253:48   0   50G  0 disk
├─vdd1          253:49   0   25G  0 part
│ └─fcache-dev1 252:0    0  100G  0 dm
└─vdd2          253:50   0   25G  0 part
  └─fcache-dev2 252:1    0  100G  0 dm

查看flashcache device

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# ll /dev/mapper/fcache-dev*
lrwxrwxrwx. 1 root root 7 Feb 11 11:14 /dev/mapper/fcache-dev1 -> ../dm-0
lrwxrwxrwx. 1 root root 7 Feb 11 11:15 /dev/mapper/fcache-dev2 -> ../dm-1

# dmsetup table
fcache-dev2: 0 209715200 flashcache conf:
	ssd dev (/dev/vdd2), disk dev (/dev/vdc) cache mode(WRITE_BACK)
	capacity(25498M), associativity(512), data block size(4K) metadata block size(4096b)
	disk assoc(0K)
	skip sequential thresh(0K)
	total blocks(6527488), cached blocks(259), cache percent(0)
	dirty blocks(0), dirty percent(0)
	nr_queued(0)
Size Hist: 4096:1497
fcache-dev1: 0 209713152 flashcache conf:
	ssd dev (/dev/vdd1), disk dev (/dev/vdb1) cache mode(WRITE_BACK)
	capacity(25498M), associativity(512), data block size(4K) metadata block size(4096b)
	disk assoc(0K)
	skip sequential thresh(0K)
	total blocks(6527488), cached blocks(259), cache percent(0)
	dirty blocks(0), dirty percent(0)
	nr_queued(0)
Size Hist: 4096:1497

使用flashcache device

创建后的flashcache device可以像普通device一样使用，如下：

1
2
3
4
5
# mkfs.ext4 /dev/mapper/fcache-dev1
# mount /dev/mapper/fcache-dev1 /mnt/
# mount | grep mnt
/dev/mapper/fcache-dev1 on /mnt type ext4 (rw,relatime,seclabel,data=ordered)
# umount /mnt/

查看flashcache device的flashcache状态

在使用一段时间后，我们可以看flashcache的缓存状态如下，可以以此为依据来调整flashcache的策略。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# dmsetup status fcache-dev1
0 209713152 flashcache stats:
	reads(1063), writes(422824)
	read hits(786), read hit percent(73)
	write hits(291) write hit percent(0)
	dirty write hits(44) dirty write hit percent(0)
	replacement(0), write replacement(7392)
	write invalidates(0), read invalidates(2)
	pending enqueues(2), pending inval(2)
	metadata dirties(422499), metadata cleans(112914)
	metadata batch(531237) metadata ssd writes(4432)
	cleanings(112914) fallow cleanings(0)
	no room(31) front merge(3042) back merge(109407)
	force_clean_block(0)
	disk reads(277), disk writes(112945) ssd reads(113700) ssd writes(427505)
	uncached reads(2), uncached writes(31), uncached IO requeue(0)
	disk read errors(0), disk write errors(0) ssd read errors(0) ssd write errors(0)
	uncached sequential reads(0), uncached sequential writes(0)
	pid_adds(0), pid_dels(0), pid_drops(0) pid_expiry(0)
	lru hot blocks(3263744), lru warm blocks(3263744)
	lru promotions(0), lru demotions(0)

查看flashcache device相关信息

1
2
3
4
5
6
7
# ls /proc/flashcache/vdd1+vdb1/
flashcache_errors  flashcache_iosize_hist  flashcache_pidlists  flashcache_stats
# cat /proc/flashcache/vdd1+vdb1/flashcache_errors
disk_read_errors=0 disk_write_errors=0 ssd_read_errors=0 ssd_write_errors=0 memory_alloc_errors=0
# cat /proc/flashcache/vdd1+vdb1/flashcache_pidlists
Blacklist:
Whitelist:

sysctl查看flashcache的信息：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
# sysctl -a | grep flashcache
dev.flashcache.vdd1+vdb1.cache_all = 1
dev.flashcache.vdd1+vdb1.clean_on_read_miss = 0
dev.flashcache.vdd1+vdb1.clean_on_write_miss = 0
dev.flashcache.vdd1+vdb1.dirty_thresh_pct = 20
dev.flashcache.vdd1+vdb1.do_pid_expiry = 0
dev.flashcache.vdd1+vdb1.do_sync = 0
dev.flashcache.vdd1+vdb1.fallow_clean_speed = 2
dev.flashcache.vdd1+vdb1.fallow_delay = 900
dev.flashcache.vdd1+vdb1.fast_remove = 0
dev.flashcache.vdd1+vdb1.io_latency_hist = 0
dev.flashcache.vdd1+vdb1.lru_hot_pct = 75
dev.flashcache.vdd1+vdb1.lru_promote_thresh = 2
dev.flashcache.vdd1+vdb1.max_clean_ios_set = 2
dev.flashcache.vdd1+vdb1.max_clean_ios_total = 4
dev.flashcache.vdd1+vdb1.max_pids = 100
dev.flashcache.vdd1+vdb1.new_style_write_merge = 0
dev.flashcache.vdd1+vdb1.pid_expiry_secs = 60
dev.flashcache.vdd1+vdb1.reclaim_policy = 0
dev.flashcache.vdd1+vdb1.skip_seq_thresh_kb = 0
dev.flashcache.vdd1+vdb1.stop_sync = 0
dev.flashcache.vdd1+vdb1.zero_stats = 0

# cd /proc/sys/dev/flashcache/vdd1+vdb1/
# ls
cache_all            dirty_thresh_pct  fallow_clean_speed  io_latency_hist     max_clean_ios_set    new_style_write_merge  skip_seq_thresh_kb
clean_on_read_miss   do_pid_expiry     fallow_delay        lru_hot_pct         max_clean_ios_total  pid_expiry_secs        stop_sync
clean_on_write_miss  do_sync           fast_remove         lru_promote_thresh  max_pids             reclaim_policy         zero_stats

不明确上述参数是否可以动态调整？

删除flashcache device

1
2
3
4
5
6
7
8
9
10
11
12
13
# dmsetup info /dev/dm-0
Name:              fcache-dev1
State:             ACTIVE
Read Ahead:        8192
Tables present:    LIVE
Open count:        0
Event number:      0
Major, minor:      252, 0
Number of targets: 1

# dmsetup remove /dev/dm-0
# flashcache_destroy /dev/vdd1
flashcache_destroy: Destroying Flashcache found on /dev/vdd1. Any data will be lost !!

遇到的问题

删除设备出错

测试中在remove一个device mapper设备时，报“Device or resource busy”。

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
# dmsetup remove /dev/dm-1
device-mapper: remove ioctl on fcache-dev2 failed: Device or resource busy
Command failed

# dmsetup info /dev/dm-1
Name:              fcache-dev2
State:             ACTIVE
Read Ahead:        256
Tables present:    LIVE
Open count:        1
Event number:      0
Major, minor:      252, 1
Number of targets: 1

# dmsetup table
fcache-dev2: 0 209715200 flashcache conf:
    ssd dev (/dev/vdd2), disk dev (/dev/vdc) cache mode(WRITE_BACK)
    capacity(25498M), associativity(512), data block size(4K) metadata block size(4096b)
    disk assoc(0K)
    skip sequential thresh(0K)
    total blocks(6527488), cached blocks(259), cache percent(0)
    dirty blocks(0), dirty percent(0)
    nr_queued(0)
Size Hist: 1024:2 4096:471740

# lsof /dev/vdd2
# lsof /dev/vdc
# lsof /dev/dm-1

尝试remove --force参数：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# dmsetup remove /dev/dm-1 --force
device-mapper: remove ioctl on fcache-dev2 failed: Device or resource busy
Command failed
# dmsetup table
fcache-dev2: 0 209715200 error

# lsblk
NAME        MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
...
vdb         253:16   0  100G  0 disk
└─vdb1      253:17   0  100G  0 part
vdc         253:32   0  100G  0 disk
vdd         253:48   0   50G  0 disk
├─vdd1      253:49   0   25G  0 part
└─vdd2      253:50   0   25G  0 part
fcache-dev2 252:1    0  100G  0 dm			## 出些在这里了???

之后尝试很多命令，没法完全把该设备移除，只能重启，重启后一切正常。

创建设备出错1

创建flashcache device时报错：“Valid Flashcache already exists on…”。

1
2
3
4
5
# flashcache_create -p back fcache-dev1 /dev/vdd1 /dev/vdb1
cachedev fcache-dev1, ssd_devname /dev/vdd1, disk_devname /dev/vdb1 cache mode WRITE_BACK
block_size 8, md_block_size 8, cache_size 0
flashcache_create: Valid Flashcache already exists on /dev/vdd1
flashcache_create: Use flashcache_destroy first and then create again /dev/vdd1

这个出错比较明确，按照提示先销毁cache设备中的flashcache数据即可。

1
2
3
4
5
6
7
# flashcache_destroy /dev/vdd1
flashcache_destroy: Destroying Flashcache found on /dev/vdd1. Any data will be lost !!

# flashcache_create -p back fcache-dev1 /dev/vdd1 /dev/vdb1
cachedev fcache-dev1, ssd_devname /dev/vdd1, disk_devname /dev/vdb1 cache mode WRITE_BACK
block_size 8, md_block_size 8, cache_size 0
Flashcache metadata will use 137MB of your 3951MB main memory

创建设备出错2

测试中反复操作，创建flashcache device时报错：“Device or resource busy”。

1
2
3
4
5
6
7
# flashcache_create -p back fcache-dev2 /dev/vdd2 /dev/vdc
cachedev fcache-dev2, ssd_devname /dev/vdd2, disk_devname /dev/vdc cache mode WRITE_BACK
block_size 8, md_block_size 8, cache_size 0
Flashcache metadata will use 137MB of your 3951MB main memory
device-mapper: reload ioctl on fcache-dev2 failed: Device or resource busy
Command failed
echo 0 209715200 flashcache /dev/vdc /dev/vdd2 fcache-dev2 1 2 8 0 512 0 0 8 | dmsetup create fcache-dev2 failed

尝试销毁flashcache设备上的相关数据，之后创建还是报错：

1
2
# flashcache_destroy /dev/vdd2
flashcache_destroy: No valid Flashcache found on /dev/vdd2

再重启机器后，创建flashcache就正常了，比较奇怪 ;(

重启机器后设备消失

重启机器后，看不到之前创建的flashcache device。

这是因为flashcache重启后不会自动加载设备，需要手动执行命令：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# modprobe flashcache
# flashcache_load
Usage: flashcache_load ssd_devname [cachedev]
git commit:
# flashcache_load /dev/vdd1

# dmsetup info
Name:              fcache-dev1
State:             ACTIVE
Read Ahead:        8192
Tables present:    LIVE
Open count:        0
Event number:      0
Major, minor:      252, 0
Number of targets: 1

若想flashcache和设备在机器重启后自动加载，可以把相关命令加入“/etc/rc.d/rc.local”。

/etc/rc.d/init.d/里也添加服务也是一个好方法 ;)

Posted on | In |

概述

之前文章大致做了LVM简介，推荐在使用本地存储时，使用LVM，那如何使用LVM呢？

这里从LVM相关组件分析其常用命令，最后给出创建一个LVM volume的步骤。

物理卷相关

物理卷 - PhysicalVolume

LVM管理的最底层就是一个个物理卷，与之相关的命令基本都以pv开始，大致有如下这些：

1
2
root@ubuntu:~# pv
pvchange   pvck       pvcreate   pvdisplay  pvmove     pvremove   pvresize   pvs        pvscan

创建

pvcreate - initialize a disk or partition for use by LVM

从物理盘或者一个分区初始化一个PV，这个PV之后会被LVM用来创建VG；

删除

pvremove - remove a physical volume

删除一个PV，之后LVM将不认为该设备为PV；

显示

pvs - report information about physical volumes

展示系统PVS的信息，可以指定输出格式等；

pvdisplay - display attributes of a physical volume

展示一个/多个PVS的属性信息，可以指定输出格式等；

其他

pvscan - scan all disks for physical volumes

扫描整个系统中支持LVM块设备的PVS；

pvresize - resize a disk or partition in use by LVM2

调整LVM使用的一个PV的大小，前提是它对应的物理设备大小可以resize；

pvchange - change attributes of a physical volume

修改一个PV的属性；

卷组相关

卷组 - VolumeGroup

卷组是LVM基于PVS创建的一个块设备集合，类似于有的存储中说的Volume Pool，它综合了底层所有PVS的存储，做了统一的配置，比如：分配块大小等；

与卷组相关的命令都是以vg开始，大致如下：

1
2
3
root@ubuntu:~# vg
vgcfgbackup    vgchange       vgconvert      vgdisplay      vgextend       vgimportclone  vgmknodes      vgremove       vgs            vgsplit
vgcfgrestore   vgck           vgcreate       vgexport       vgimport       vgmerge        vgreduce       vgrename       vgscan

创建

vgcreate - create a volume group

基于之前创建的PVS创建一个VG，这里原则上选择同质物理盘创建的PVS；

可以指定每次在PV上申请空间的Extent size（-s, –physicalextentsize PhysicalExtentSize[bBsSkKmMgGtTpPeE]）

删除

vgremove - remove a volume group

删除一个VG；若VG上有LV，则会有相关提示；

显示

vgs - report information about volume groups

显示系统中有的VGS信息；

vgdisplay - display attributes of volume groups

显示VGS的详细属性信息；

其他

vgextend - add physical volumes to a volume group

添加PVS到VG中，扩容VG；

vgreduce - reduce a volume group

从VG中删除没有使用的PVS，若PV已经分配了PE，则命令失败；

vgchange - change attributes of a volume group

修改一个VG的属性；

逻辑卷相关

逻辑卷 - LogicalVolume

逻辑卷是最终面向用户使用的块设备，用户可以像使用普通磁盘/分区一样的使用它；同时逻辑卷支持很多普通磁盘没有的特性，比如：mirror，stripe，raid，thin等；

与逻辑卷相关的命令都是以lv开始，大致如下：

1
2
3
root@ubuntu:~# lv
lvchange     lvcreate     lvextend     lvmchange    lvmconfig    lvmdump      lvmpolld     lvmsar       lvremove     lvresize     lvscan
lvconvert    lvdisplay    lvm          lvmconf      lvmdiskscan  lvmetad      lvmsadc      lvreduce     lvrename     lvs

创建

lvcreate - create a logical volume in an existing volume group

从一个创建好的VG里创建一个LV，可以指定LV的很多属性，比如：size，stripe，raid，thin等；

删除

lvremove - remove a logical volume

删除一个LV；若该LV系统正在使用，则删除失败；

显示

lvs - report information about logical volumes

显示系统LVS的基本信息；

lvdisplay - display attributes of a logical volume

显示一个LV的详细属性信息；

其他

lvextend - extend the size of a logical volume

扩容一个LV的size；

lvresize - resize a logical volume

调整一个LV的size；可以扩容，可以缩容，但缩容需谨慎，可能导致数据丢失；

lvchange - change attributes of a logical volume

修改一个LV的属性；

lvscan - scan (all disks) for Logical Volumes

扫描系统获取所有LV的 active，snapshot/origin，size等信息；

创建LVM Volume步骤

创建PV

使用fdisk创建LVM Type的磁盘分区

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
[root@ceph0 ~]# fdisk /dev/sdj
Command (m for help): n
Partition number (1-128, default 1):
First sector (34-7814037134, default 2048):
Last sector, +sectors or +size{K,M,G,T,P} (2048-7814037134, default 7814037134):
Created partition 1
 
 
Command (m for help): t
Selected partition 1
Partition type (type L to list all types): 15
Changed type of partition 'Linux filesystem' to 'Linux LVM'
 
Command (m for help): p
 
Disk /dev/sdj: 4000.8 GB, 4000787030016 bytes, 7814037168 sectors
Units = sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk label type: gpt
 
 
#         Start          End    Size  Type            Name
 1         2048   7814037134    3.7T  Linux LVM
 
 
Command (m for help): w
The partition table has been altered!
 
Calling ioctl() to re-read partition table.
Syncing disks.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
[root@ceph0 ~]# pvs
  PV         VG Fmt  Attr PSize   PFree
  /dev/sdi1     lvm2 ---    3.64t 3.64t
  /dev/sdj1     lvm2 ---    3.64t 3.64t
  /dev/sdk1     lvm2 ---    3.64t 3.64t
  /dev/sdl1     lvm2 ---    3.64t 3.64t
 
 
[root@ceph0 ~]# pvdisplay /dev/sdi1
  --- Physical volume ---
  PV Name               /dev/sdi1
  VG Name               volgroup-sata
  PV Size               3.64 TiB / not usable 2.82 MiB
  Allocatable           yes
  PE Size               4.00 MiB
  Total PE              953861
  Free PE               953605
  Allocated PE          256
  PV UUID               rGiKHq-yem2-DUSH-F5AM-4jxE-eZvv-cF1YlG

创建VG

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
[root@ceph0 ~]# vgcreate volgroup-sata /dev/sdi1 /dev/sdj1 /dev/sdk1
  Volume group "volgroup-sata" successfully created
[root@ceph0 ~]#
[root@ceph0 ~]# vgs
  VG            #PV #LV #SN Attr   VSize   VFree
  volgroup-sata   3   0   0 wz--n-  10.92t 10.92t
 
 
[root@ceph0 ~]# vgdisplay volgroup-sata
  --- Volume group ---
  VG Name               volgroup-sata
  System ID
  Format                lvm2
  Metadata Areas        3
  Metadata Sequence No  9
  VG Access             read/write
  VG Status             resizable
  MAX LV                0
  Cur LV                1
  Open LV               0
  Max PV                0
  Cur PV                3
  Act PV                3
  VG Size               10.92 TiB
  PE Size               4.00 MiB
  Total PE              2861583
  Alloc PE / Size       256 / 1.00 GiB
  Free  PE / Size       2861327 / 10.92 TiB
  VG UUID               7rEMc5-pkzR-ckFM-8Yve-cATM-uy9B-buMcR6

创建LV

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
[root@ceph0 ~]# lvcreate -n lv1 -L 1G volgroup-sata
  Logical volume "lv1" created.
[root@ceph0 ~]# lvs
  LV   VG            Attr       LSize   Pool Origin Data%  Meta%  Move Log Cpy%Sync Convert
  lv1  volgroup-sata -wi-a-----   1.00g
 
 
[root@ceph0 ~]# lvdisplay volgroup-sata/lv1
  --- Logical volume ---
  LV Path                /dev/volgroup-sata/lv1
  LV Name                lv1
  VG Name                volgroup-sata
  LV UUID                SWp2Lr-kGLC-8USu-3jqu-Bwpd-yN3D-wMGFGn
  LV Write Access        read/write
  LV Creation host, time ceph0, 2018-01-26 16:19:17 +0800
  LV Status              available
  # open                 0
  LV Size                1.00 GiB
  Current LE             256
  Segments               1
  Allocation             inherit
  Read ahead sectors     auto
  - currently set to     256
  Block device           253:3

使用LV

可以像使用系统物理device一样，使用LogicalVolume；

1
2
[root@ceph0 ~]# mkfs.ext4 /dev/volgroup-sata/lv1
[root@ceph0 ~]# mount /dev/volgroup-sata/lv1 /mnt/

Posted on | In |

LVM基本组成

LVM利用Linux内核的device-mapper来实现存储系统的虚拟化（系统分区独立于底层硬件）。 通过LVM，你可以实现存储空间的抽象化并在上面建立虚拟分区（virtual partitions），可以更简便地扩大和缩小分区，可以增删分区时无需担心某个硬盘上没有足够的连续空间， LVM是用来方便管理的，不会提供额外的安全保证。

LVM的基本组成块（building blocks）如下：

  • 物理卷Physical volume (PV)：可以在上面建立卷组的媒介，可以是硬盘分区，也可以是硬盘本身或者回环文件（loopback file）。物理卷包括一个特殊的header，其余部分被切割为一块块物理区域（physical extents）。
  • 卷组Volume group (VG)：将一组物理卷收集为一个管理单元。
  • 逻辑卷Logical volume (LV)：虚拟分区，由物理区域（physical extents）组成。
  • 物理区域Physical extent (PE)：硬盘可供指派给逻辑卷的最小单位（通常为4MB）。

示例:

1
2
3
4
5
6
7
8
9
10
11
12
13
两块物理硬盘
                
  硬盘1 (/dev/sda):
     _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
    |分区1 50GB (物理卷)            |分区2 80GB (物理卷)            |
    |/dev/sda1                    |/dev/sda2                     |
    |_ _ _ _ _ _ _ _ _ _ _ _ _ _ _|_ _ _ _ _ _ _ _ _ _ _ _ _ _ __|
                                  
  硬盘2 (/dev/sdb):
     _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
    |分区1 120GB (物理卷)                          |
    |/dev/sdb1                                   |
    | _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _|
1
2
3
LVM方式

  卷组VG1 (/dev/storage1/ = /dev/sda1):
1
卷组VG2 (/dev/storage2/ = /dev/sda2 + /dev/sdb1):
1
2
3
4
 _ _ _ _ _ _ _  _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
|逻辑卷1 15GB             |逻辑卷2 35GB               |逻辑卷3 200GB               |
|/dev/storage1/rootvol   |/dev/storage2/homevol     |/dev/storage2/mediavol     |
|_ _ _ _ _ _ _ _ _ _ _ _ |_ _ _ _ _ _ _ _ _ _ _ _ _ |_ _ _ _ _ _ _ _ _ _ _ _ _ _|

图解如下

lvm-arch

优点

比起正常的硬盘分区管理，LVM更富于弹性：

  • 使用卷组(VG)，使众多硬盘空间看起来像一个大硬盘。
  • 使用逻辑卷（LV），可以创建跨越众多硬盘空间的分区。
  • 可以创建小的逻辑卷（LV），在空间不足时再动态调整它的大小。
  • 在调整逻辑卷（LV）大小时可以不用考虑逻辑卷在硬盘上的位置，不用担心没有可用的连续空间。It does not depend on the position of the LV within VG, there is no need to ensure surrounding available space.
  • 可以在线（online）对逻辑卷（LV）和卷组（VG）进行创建、删除、调整大小等操作。LVM上的文件系统也需要重新调整大小，某些文件系统也支持这样的在线操作。
  • 无需重新启动服务，就可以将服务中用到的逻辑卷（LV）在线（online）/动态（live）迁移至别的硬盘上。
  • 允许创建快照，可以保存文件系统的备份，同时使服务的下线时间（downtime）降低到最小。

这些优点使得LVM对服务器的管理非常有用，对于桌面系统管理的帮助则没有那么显著，你需要根据实际情况进行取舍。

缺点

  • 在系统设置时需要更复杂的额外步骤。

系统支持

当前Linux系统都支持LVM，我们常用的Ubuntu和Centos上都可以方便安装，具体如下：

Ubuntu：

1
2
3
4
5
6
7
8
9
10
11
12
13
root@ubuntu:~# lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description:    Ubuntu 16.04.2 LTS
Release:    16.04
Codename:   xenial
 
root@ubuntu:~# apt-get install -y lvm2
...
root@ubuntu:~# lvm version
  LVM version:     2.02.133(2) (2015-10-30)
  Library version: 1.02.110 (2015-10-30)
  Driver version:  4.34.0

Centos：

1
2
3
4
5
6
7
8
9
10
11
12
13
root@centos:~# lsb_release -a
LSB Version:    :core-4.1-amd64:core-4.1-noarch:cxx-4.1-amd64:cxx-4.1-noarch:desktop-4.1-amd64:desktop-4.1-noarch:languages-4.1-amd64:languages-4.1-noarch:printing-4.1-amd64:printing-4.1-noarch
Distributor ID: CentOS
Description:    CentOS Linux release 7.3.1611 (Core)
Release:    7.3.1611
Codename:   Core
 
root@centos:~# yum install -y lvm2
...
root@centos:~# lvm version
  LVM version:     2.02.171(2)-RHEL7 (2017-05-03)
  Library version: 1.02.140-RHEL7 (2017-05-03)
  Driver version:  4.34.0

使用建议

使用本地存储时，LVM是对比物理盘有明显的优点，建议除非特别简单使用物理盘的场景，都可以使用LVM来提供存储服务。

使用时候的一些建议如下：

  1. 一个物理盘对应一个PV
  2. 多个PV构成一个VG
  3. 同一VG内最好使用相同性能的物理盘
  4. 不同性能的物理盘，创建不同的VG
  5. 创建适当大小的LV，容量不够时再扩容

Posted on | In |

概述

我们知道LVM是管理本地磁盘，更好的提供块设备服务的机制；

在现实环境中，我们的机器通常都有几种不同类型的磁盘存在，高性能的磁盘容量低，容量大的磁盘性能差，那如何利用Linux系统的各种cache机制来提升LVM卷的性能呢？

存储磁盘分类

通常我们使用的存储磁盘有三种，分别对应不同的容量、性能、价格，有：SATA盘，SSD盘和PCIE-SSD；

他们三个的大致性能容量和价格如下表所示：

设备 容量 性能 BandWidth IOPS 价格 厂商 应用场景
SATA 容量大，常见为4-8TB 低 约100MB/s 100 低 希捷，西数等 大容量
SSD 容量小，常见为几百GB 高 约500MB/s 读：30, 000+写：5, 000+ 高 Intel，Samsung等 高性能，加速应用
PCIE-SSD 容量较大，常见为1-4T 超高 约2GB/s 读：500, 000+写：100, 000+ 较高 Intel，Samsung，Memblaze，宝存等 超高性能，加速应用

从上面可以看出，可以作为普通磁盘加速的磁盘有：SSD盘和PCIE-SSD，其中SSD盘是最常见和用的最多的，PCIE卡在超高性能需求场所有使用。

注：其实现在SSD/PCIE-SSD的每GB价格相差不大了，若对超高性能有需求，PCIE-SSD是个很好的选择；

Cache算法

通常我们使用高性能磁盘来加速低性能磁盘时，都会选择一种Linux内核支持的Cache算法，现在Linux内核支持的常用Cache算法有如下几种；

flashcache

flashcache 是 facebook 开源的 ssd 存储产品，它基于内核的 devicemapper 机制，允许将 ssd 设备映射为机械存储设备的缓存，堆叠成为一个虚拟设备供用户读写，从而在一定程度上兼顾 ssd 的高速与机械存储设备的高容量，更加经济高效地支撑线上业务。

flashcache 支持三种缓存策略：

  • 回写(Write Back)：修改内容之后，并不立即写入后端设备
  • 写透(Write Through): 修改内容时写入后端设备，同时也更新前端设备中的缓存块
  • 环写(Write Around): 修改内容时，先写入后端设备，同时使前端设备中对应的缓存块失效

参考：

https://github.com/facebookarchive/flashcache

https://en.wikipedia.org/wiki/Flashcache

bcache

bcache是linux内核块设备层cache，类似于flashcache使用ssd作为hdd的缓存方案，相比于flashcache，bcache更加灵活，支持ssd作为多块hdd的共享缓存，并且还支持多块ssd（还未完善），能够在运行中动态增加，删除缓存设备和后端设备。

从3.10开始，bcache进入内核主线。bcache支持writeback、writethrough、writearoud三种策略，默认是wriththrough，可以动态修改，缓存替换方式支持lru、fifo和random三种。

参考：

https://bcache.evilpiepirate.org/

http://www.sysnote.org/2014/06/20/bcache-analysis/

dm-cache

作为linux内核的一部分，dm-cache采用device mapper机制允许用户建立混合卷。dm-cache可以采用一个或多个快速设备为后端慢速存储系统扮演缓存的角色。

dm-cache设计成由3个物理存储设备来混合成一个逻辑卷的形式。操作模式和缓存策略决定了缓存数据的性能。这三个物理设备分别为：

  • 原始设备：提供主要的慢速存储（通常是一个硬盘或者SAN）
  • 缓存设备：提供高速原始设备数据的缓存（通常是一个SSD）
  • 元数据设备：记录硬盘块在缓存中的位置，脏标志以及执行缓存策略所需的内部数据

dm-cache支持写回，写通和旁路三种模式。

参考：

https://en.wikipedia.org/wiki/Dm-cache

LVM加速策略

flashcache

基于flashcache的使用策略（SSD-SATA 1对1），我们有如下策略：

lvm-cahce-flashcache

bcache

bcache可以用一块SSD盘做多个SATA盘的缓存，有如下策略：

lvm-cahce-bcache

LVM自带Cache策略

LVM自带Cache策略可以配置两个LV之间做缓存，有如下策略：

lvm-cahce-self

LVM Cache策略选择

现实中，考虑稳定性和可靠性，使用flashcache作为Cache策略的比较多。

但最近bcache也越来越得到大范围的使用，在某些测试报告中，其性能多数场景优于flashcache。

Posted on | In |

概述

最近有需求要分析定位下开发的文件存储网关的内存泄露问题，对比了几款后选择了valgrind这款工具，功能很是强大，这里我还只使用了一些基本的功能，记录如下。

Valgrind支持很多工具: Memcheck，Addrcheck，Cachegrind，Massif，Helgrind和Callgrind等。

官网：http://valgrind.org/

安装

1
2
3
4
5
6
7
$ wget ftp://sourceware.org/pub/valgrind/valgrind-3.13.0.tar.bz2
$ bzip2 -d valgrind-3.13.0.tar.bz2
$ tar -xf valgrind-3.13.0.tar
$ cd valgrind-3.13.0/
$ ./configure
$ make
$ sudo make install

使用与分析

1
2
3
4
5
6
7
8
9
10
11
12
13
14
$ valgrind --help
usage: valgrind [options] prog-and-args
...

常用的一些options有：
--trace-children=no|yes   Valgrind-ise child processes (follow execve)? [no]
--log-file=<file>         log messages to <file>
--leak-check=no|summary|full     search for memory leaks at exit?  [summary]

--show-reachable=yes             same as --show-leak-kinds=all
    --show-reachable=no --show-possibly-lost=yes
                                     same as --show-leak-kinds=definite,possible
    --show-reachable=no --show-possibly-lost=no
                                     same as --show-leak-kinds=definite

常用命令格式：

1
$ /usr/local/bin/valgrind --log-file=mylog ./file-gateway

valgrind的memcheck组件支持的内存泄露检查大概有：

  1. 读写非法的内存
  2. 访问未初始化的内存
  3. 访问已经释放了的内存
  4. 申请的内存没有释放
  5. 重复释放内存
  6. 释放非法内存
  7. source和destination内存的重叠

uninitialised byte(s)

检查出程序中未初始化而访问的变量

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
==1350634== Thread 14:
==1350634== Syscall param setxattr(value) points to uninitialised byte(s)
==1350634==    at 0x6D9EB8A: setxattr (in /usr/lib64/libc-2.17.so)
==1350634==    by 0x487268: VFS::file_setxattr(std::string, std::string, void*, int) (vfs.cc:64)
==1350634==    by 0x488066: FileGwVFS::file_setxattr(std::string, std::string, void*, int) (vfs.cc:175)
==1350634==    by 0x459583: FileGateway::set_file_xattr(std::string, sgwd_xattr*) (file_gateway.cc:742)
==1350634==    by 0x463706: FileGwRedisWorkItem::update_file_xattr_status(gateway_file_status) (redis_work_item.cc:703)
==1350634==    by 0x4641B1: FileGwRedisWorkItem::file_upload() (redis_work_item.cc:793)
==1350634==    by 0x45F89E: FileGwRedisWorkItem::process() (redis_work_item.cc:190)
==1350634==    by 0x486532: WorkThread::process_work_item() (work_thread.cc:49)
==1350634==    by 0x486850: ShardedWorkThread::entry() (work_thread.cc:112)
==1350634==    by 0x481B58: Thread::_entry_func(void*) (thread.cc:74)
==1350634==    by 0x4E3DDC4: start_thread (in /usr/lib64/libpthread-2.17.so)
==1350634==    by 0x6D9FCEC: clone (in /usr/lib64/libc-2.17.so)
==1350634==  Address 0x115d95a4 is on thread 14's stack
==1350634==  in frame #4, created by FileGwRedisWorkItem::update_file_xattr_status(gateway_file_status) (redis_work_item.cc:697)
1
2
3
4
5
6
7
8
9
==1350634== Thread 19:
==1350634== Conditional jump or move depends on uninitialised value(s)
==1350634==    at 0x465155: FileGwRedisWorkItem::callback(void*) (redis_work_item.cc:946)
==1350634==    by 0x481021: FileGwS3WorkItem::postProcess() (s3_work_item.cc:244)
==1350634==    by 0x486551: WorkThread::process_work_item() (work_thread.cc:50)
==1350634==    by 0x48666E: WorkThread::entry() (work_thread.cc:78)
==1350634==    by 0x481B58: Thread::_entry_func(void*) (thread.cc:74)
==1350634==    by 0x4E3DDC4: start_thread (in /usr/lib64/libpthread-2.17.so)
==1350634==    by 0x6D9FCEC: clone (in /usr/lib64/libc-2.17.so)

Invalid read/write

读取非法地址，通常是继续访问释放后的内存空间

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
==1346085== Thread 14:
==1346085== Invalid read of size 4
==1346085==    at 0x4E40BB0: pthread_mutex_unlock (in /usr/lib64/libpthread-2.17.so)
==1346085==    by 0x4559FC: __gthread_mutex_unlock(pthread_mutex_t*) (gthr-default.h:778)
==1346085==    by 0x45B6C1: std::mutex::unlock() (mutex:152)
==1346085==    by 0x489CA8: std::lock_guard<std::mutex>::~lock_guard() (mutex:420)
==1346085==    by 0x4895BB: WorkItemCompletion::finish() (thread_work_item.cc:47)
==1346085==    by 0x48971E: ThreadWorkItem::postProcess() (thread_work_item.cc:70)
==1346085==    by 0x45F721: RedisWorkItem::postProcess() (redis_work_item.cc:139)
==1346085==    by 0x45FFED: FileGwRedisWorkItem::postProcess() (redis_work_item.cc:234)
==1346085==    by 0x486761: WorkThread::process_work_item() (work_thread.cc:50)
==1346085==    by 0x486A60: ShardedWorkThread::entry() (work_thread.cc:112)
==1346085==    by 0x481D68: Thread::_entry_func(void*) (thread.cc:74)
==1346085==    by 0x4E3DDC4: start_thread (in /usr/lib64/libpthread-2.17.so)
==1346085==  Address 0xabd3788 is 24 bytes inside a block of size 88 free'd
==1346085==    at 0x4C2B1CD: operator delete(void*) (vg_replace_malloc.c:576)
==1346085==    by 0x489253: WorkItemCompletion::~WorkItemCompletion() (thread_work_item.cc:29)
==1346085==    by 0x4572E4: FileGateway::issue_filegw_redis_workitem(FileOp*, FileInfo*, bool, bool, void (*)(void*), void*) (file_gateway.cc:298)
==1346085==    by 0x457D31: FileGateway::do_read(FileOp*) (file_gateway.cc:455)
==1346085==    by 0x455298: GatewayWorkItem::process() (gateway_work_item.cc:74)
==1346085==    by 0x486742: WorkThread::process_work_item() (work_thread.cc:49)
==1346085==    by 0x48687E: WorkThread::entry() (work_thread.cc:78)
==1346085==    by 0x481D68: Thread::_entry_func(void*) (thread.cc:74)
==1346085==    by 0x4E3DDC4: start_thread (in /usr/lib64/libpthread-2.17.so)
==1346085==    by 0x6D9FCEC: clone (in /usr/lib64/libc-2.17.so)
==1346085==  Block was alloc'd at
==1346085==    at 0x4C2A243: operator new(unsigned long) (vg_replace_malloc.c:334)
==1346085==    by 0x457171: FileGateway::issue_filegw_redis_workitem(FileOp*, FileInfo*, bool, bool, void (*)(void*), void*) (file_gateway.cc:286)
==1346085==    by 0x457D31: FileGateway::do_read(FileOp*) (file_gateway.cc:455)
==1346085==    by 0x455298: GatewayWorkItem::process() (gateway_work_item.cc:74)
==1346085==    by 0x486742: WorkThread::process_work_item() (work_thread.cc:49)
==1346085==    by 0x48687E: WorkThread::entry() (work_thread.cc:78)
==1346085==    by 0x481D68: Thread::_entry_func(void*) (thread.cc:74)
==1346085==    by 0x4E3DDC4: start_thread (in /usr/lib64/libpthread-2.17.so)
==1346085==    by 0x6D9FCEC: clone (in /usr/lib64/libc-2.17.so)

SUMMARY

在valgrind最后会有内存分析的总结

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
==1366931== HEAP SUMMARY:
==1366931==     in use at exit: 41,110 bytes in 1,019 blocks
==1366931==   total heap usage: 1,275,222 allocs, 1,274,203 frees, 334,495,820 bytes allocated
==1366931==
==1366931== LEAK SUMMARY:
==1366931==    definitely lost: 11,632 bytes in 191 blocks
==1366931==    indirectly lost: 80 bytes in 2 blocks
==1366931==      possibly lost: 608 bytes in 1 blocks
==1366931==    still reachable: 28,790 bytes in 825 blocks
==1366931==                       of which reachable via heuristic:
==1366931==                         stdstring          : 145 bytes in 4 blocks
==1366931==         suppressed: 0 bytes in 0 blocks
==1366931== Rerun with --leak-check=full to see details of leaked memory
==1366931==
==1366931== For counts of detected and suppressed errors, rerun with: -v
==1366931== ERROR SUMMARY: 0 errors from 0 contexts (suppressed: 0 from 0)

LEAK SUMMARY：内存泄露信息的总结，详细信息从前面的log中查找

  • definitely lost: 肯定丢失的内存，这部分必须处理
  • indirectly lost: 间接丢失的内存，可能是一个structure里指向的内存
  • possibly lost: 可能丢失的部分，这是由于C/C++语言指针处理的特点造成的，可能不太准确
  • still reachable: 程序可能是ok的，可以配置--show-reachable=no不显示这部分

若想显示泄露内存的详细信息，使用参数--leak-check=full

/usr/local/bin/valgrind --leak-check=full --log-file=mylog ./flle-gateway

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
==1373210== HEAP SUMMARY:
==1373210==     in use at exit: 30,758 bytes in 843 blocks
==1373210==   total heap usage: 405,641 allocs, 404,798 frees, 44,538,821 bytes allocated
==1373210==
==1373210== 608 bytes in 1 blocks are possibly lost in loss record 605 of 613
==1373210==    at 0x4C2B9B5: calloc (vg_replace_malloc.c:711)
==1373210==    by 0x4011DE4: _dl_allocate_tls (in /usr/lib64/ld-2.17.so)
==1373210==    by 0x4E3E960: pthread_create@@GLIBC_2.2.5 (in /usr/lib64/libpthread-2.17.so)
==1373210==    by 0x489969: libsgsvc_init (in /home/yangguanjun3/storage-file-gateway/userspace/build/storage-flle-gw)
==1373210==    by 0x456B06: FileGateway::init_netlink() (file_gateway.cc:218)
==1373210==    by 0x456CA4: FileGateway::init() (file_gateway.cc:234)
==1373210==    by 0x4537A2: main (main.cc:79)
==1373210==
==1373210== 1,360 bytes in 17 blocks are definitely lost in loss record 611 of 613
==1373210==    at 0x4C29C23: malloc (vg_replace_malloc.c:299)
==1373210==    by 0x4785C7: Aws::Utils::Stream::PreallocatedStreamBuf* Aws::New<Aws::Utils::Stream::PreallocatedStreamBuf, Aws::Utils::Array<unsigned char>*, unsigned long>(char const*, Aws::Utils::Array<unsigned char>*&&, unsigned long&&) (AWSMemory.h:70)
==1373210==    by 0x473446: S3BackendStorage::put_object_from_file_slice(std::basic_string<char, std::char_traits<char>, Aws::Allocator<char> >, std::basic_string<char, std::char_traits<char>, Aws::Allocator<char> >, std::string const&, unsigned long, unsigned long) (s3_storage.cc:294)
==1373210==    by 0x480110: FileGwS3WorkItem::file_upload() (s3_work_item.cc:147)
==1373210==    by 0x480C0D: FileGwS3WorkItem::process() (s3_work_item.cc:221)
==1373210==    by 0x486316: WorkThread::process_work_item() (work_thread.cc:49)
==1373210==    by 0x486452: WorkThread::entry() (work_thread.cc:78)
==1373210==    by 0x48193C: Thread::_entry_func(void*) (thread.cc:74)
==1373210==    by 0x4E3DDC4: start_thread (in /usr/lib64/libpthread-2.17.so)
==1373210==    by 0x6D9FCEC: clone (in /usr/lib64/libc-2.17.so)
==1373210==
==1373210== LEAK SUMMARY:
==1373210==    definitely lost: 1,360 bytes in 17 blocks
==1373210==    indirectly lost: 0 bytes in 0 blocks
==1373210==      possibly lost: 608 bytes in 1 blocks
==1373210==    still reachable: 28,790 bytes in 825 blocks
==1373210==                       of which reachable via heuristic:
==1373210==                         stdstring          : 145 bytes in 4 blocks
==1373210==         suppressed: 0 bytes in 0 blocks
==1373210== Reachable blocks (those to which a pointer was found) are not shown.
==1373210== To see them, rerun with: --leak-check=full --show-leak-kinds=all
==1373210==
==1373210== For counts of detected and suppressed errors, rerun with: -v
==1373210== ERROR SUMMARY: 2 errors from 2 contexts (suppressed: 0 from 0)

参考

http://valgrind.org/docs/manual/manual.html
http://www.oschina.net/translate/valgrind-memcheck
https://www.ibm.com/developerworks/cn/linux/l-cn-valgrind/

Posted on | In |

基于ceph jewel 10.2.5版本

参考文档：http://docs.ceph.com/docs/master/radosgw/layout/

测试数据

创建一个user：ictfox
创建这个user下的一个bucket：bruins
往bucket里写一个object：hello.txt

所以这里的测试数据有：

  • user info
  • bucket info
  • object info

注：本文描述了查询radosgw数据和元数据的方法，没有深入去解析一些命令输出；若有错误，欢迎指正 ;)

RadosGW的pools

查看radosgw对应的所有pools，其所有的数据也都存在这些pools里。

1
2
3
4
5
6
7
8
9
10
11
12
# rados lspools
.rgw.root
default.rgw.control
default.rgw.data.root
default.rgw.gc
default.rgw.log
default.rgw.users.uid
default.rgw.users.keys
default.rgw.buckets.index
default.rgw.buckets.data
default.rgw.users.swift
default.rgw.users.email

很多pools的用途通过名字就能明确，若要明确pools的用途，请参考代码: rgw/rgw_rados.cc

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
static string default_storage_pool_suffix = "rgw.buckets.data";
static string default_bucket_index_pool_suffix = "rgw.buckets.index";
static string default_storage_extra_pool_suffix = "rgw.buckets.non-ec";

static string zone_info_oid_prefix = "zone_info.";
static string zone_names_oid_prefix = "zone_names.";
static string region_info_oid_prefix = "region_info.";
static string zone_group_info_oid_prefix = "zonegroup_info.";
static string realm_names_oid_prefix = "realms_names.";
static string realm_info_oid_prefix = "realms.";
static string default_region_info_oid = "default.region";
static string default_zone_group_info_oid = "default.zonegroup";
static string period_info_oid_prefix = "periods.";
static string period_latest_epoch_info_oid = ".latest_epoch";
static string region_map_oid = "region_map";
static string zonegroup_map_oid = "zonegroup_map";
static string log_lock_name = "rgw_log_lock";
static string default_realm_info_oid = "default.realm";
const string default_zonegroup_name = "default";
const string default_zone_name = "default";
static string zonegroup_names_oid_prefix = "zonegroups_names.";

static string RGW_DEFAULT_ZONE_ROOT_POOL = "rgw.root";
static string RGW_DEFAULT_ZONEGROUP_ROOT_POOL = "rgw.root";
static string RGW_DEFAULT_REALM_ROOT_POOL = "rgw.root";
static string RGW_DEFAULT_PERIOD_ROOT_POOL = "rgw.root";
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
int RGWZoneParams::fix_pool_names()
{

    list<string> zones;
    int r = store->list_zones(zones);
    if (r < 0) {
        ldout(cct, 10) << "WARNING: store->list_zones() returned r=" << r << dendl;
    }

    set<string> pool_names;
    r = get_zones_pool_names_set(cct, store, zones, id, pool_names);
    if (r < 0) {
        ldout(cct, 0) << "Error: get_zones_pool_names" << r << dendl;
        return r;
    }

    domain_root = fix_zone_pool_name(pool_names, name, ".rgw.data.root", domain_root.name);
    if (!metadata_heap.name.empty()) {
        metadata_heap = fix_zone_pool_name(pool_names, name, ".rgw.meta", metadata_heap.name);
    }
    control_pool = fix_zone_pool_name(pool_names, name, ".rgw.control", control_pool.name);
    gc_pool = fix_zone_pool_name(pool_names, name,".rgw.gc", gc_pool.name);
    log_pool = fix_zone_pool_name(pool_names, name, ".rgw.log", log_pool.name);
    intent_log_pool = fix_zone_pool_name(pool_names, name, ".rgw.intent-log", intent_log_pool.name);
    usage_log_pool = fix_zone_pool_name(pool_names, name, ".rgw.usage", usage_log_pool.name);
    user_keys_pool = fix_zone_pool_name(pool_names, name, ".rgw.users.keys", user_keys_pool.name);
    user_email_pool = fix_zone_pool_name(pool_names, name, ".rgw.users.email", user_email_pool.name);
    user_swift_pool = fix_zone_pool_name(pool_names, name, ".rgw.users.swift", user_swift_pool.name);
    user_uid_pool = fix_zone_pool_name(pool_names, name, ".rgw.users.uid", user_uid_pool.name);

    for(auto& iter : placement_pools) {
        iter.second.index_pool = fix_zone_pool_name(pool_names, name, "." + default_bucket_index_pool_suffix,
                                 iter.second.index_pool);
        iter.second.data_pool = fix_zone_pool_name(pool_names, name, "." + default_storage_pool_suffix,
                                iter.second.data_pool);
        iter.second.data_extra_pool= fix_zone_pool_name(pool_names, name, "." + default_storage_extra_pool_suffix,
                                     iter.second.data_extra_pool);
    }

    return 0;
}

radosgw的元数据信息

通过命令radosgw-admin查看原数据信息，包括bucket信息和user信息。

user信息

radosgw-admin user info命令

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
# radosgw-admin user info --uid=ictfox
{
    "user_id": "ictfox",
    "display_name": "mike",
    "email": "",
    "suspended": 0,
    "max_buckets": 1000,
    "auid": 0,
    "subusers": [],
    "keys": [
        {
            "user": "ictfox",
            "access_key": "IYZ800MDM7VF3EDLISC7",
            "secret_key": "05HfpRrrh1Gs1p8bxBbcn2HcGp0n7UvuaMNPCuHS"
        }
    ],
    "swift_keys": [],
    "caps": [],
    "op_mask": "read, write, delete",
    "default_placement": "",
    "placement_tags": [],
    "bucket_quota": {
        "enabled": false,
        "max_size_kb": -1,
        "max_objects": -1
    },
    "user_quota": {
        "enabled": false,
        "max_size_kb": -1,
        "max_objects": -1
    },
    "temp_url_keys": []
}

radosgw-admin metadata命令

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
# radosgw-admin metadata list user
[
    "ictfox"
]
# radosgw-admin metadata get user:ictfox
{
    "key": "user:ictfox",
    "ver": {
        "tag": "_laJXG6YTAtNd-Dzt78okwU5",
        "ver": 1
    },
    "mtime": "2017-03-14 08:49:04.735466Z",
    "data": {
        "user_id": "ictfox",
        "display_name": "mike",
        "email": "",
        "suspended": 0,
        "max_buckets": 1000,
        "auid": 0,
        "subusers": [],
        "keys": [
            {
                "user": "ictfox",
                "access_key": "IYZ800MDM7VF3EDLISC7",
                "secret_key": "05HfpRrrh1Gs1p8bxBbcn2HcGp0n7UvuaMNPCuHS"
            }
        ],
        "swift_keys": [],
        "caps": [],
        "op_mask": "read, write, delete",
        "default_placement": "",
        "placement_tags": [],
        "bucket_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        },
        "user_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        },
        "temp_url_keys": [],
        "attrs": [
            {
                "key": "user.rgw.idtag",
                "val": ""
            },
            {
                "key": "user.rgw.manifest",
                "val": ""
            }
        ]
    }
}

bucket信息

radosgw-admin bucket stats命令

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# radosgw-admin bucket stats --bucket=bruins
{
    "bucket": "bruins",
    "pool": "default.rgw.buckets.data",
    "index_pool": "default.rgw.buckets.index",
    "id": "070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
    "marker": "070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
    "owner": "ictfox",
    "ver": "0#11",
    "master_ver": "0#0",
    "mtime": "2017-03-15 00:46:37.189715",
    "max_marker": "0#",
    "usage": {
        "rgw.main": {
            "size_kb": 1,
            "size_kb_actual": 4,
            "num_objects": 1
        }
    },
    "bucket_quota": {
        "enabled": false,
        "max_size_kb": -1,
        "max_objects": -1
    }
}

radosgw-admin metadata命令

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
# radosgw-admin metadata list bucket
[
    "bruins",
]
# radosgw-admin metadata get bucket:bruins
{
    "key": "bucket:bruins",
    "ver": {
        "tag": "_osgTpZcD-jCFgcwdi3yvHmH",
        "ver": 1
    },
    "mtime": "2017-03-15 04:46:37.193584Z",
    "data": {
        "bucket": {
            "name": "bruins",
            "pool": "default.rgw.buckets.data",
            "data_extra_pool": "default.rgw.buckets.non-ec",
            "index_pool": "default.rgw.buckets.index",
            "marker": "070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
            "bucket_id": "070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
            "tenant": ""
        },
        "owner": "ictfox",
        "creation_time": "0.000000",
        "linked": "true",
        "has_bucket_info": "false"
    }
}

# radosgw-admin metadata list bucket.instance
[
    "bruins:070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
]

# radosgw-admin metadata get bucket.instance:bruins:070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1
{
    "key": "bucket.instance:bruins:070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
    "ver": {
        "tag": "_n2XczjnP1PYx3Hg88iruBbm",
        "ver": 1
    },
    "mtime": "2017-03-15 04:46:37.189715Z",
    "data": {
        "bucket_info": {
            "bucket": {
                "name": "bruins",
                "pool": "default.rgw.buckets.data",
                "data_extra_pool": "default.rgw.buckets.non-ec",
                "index_pool": "default.rgw.buckets.index",
                "marker": "070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
                "bucket_id": "070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
                "tenant": ""
            },
            "creation_time": "0.000000",
            "owner": "ictfox",
            "flags": 0,
            "zonegroup": "d40f14ea-4700-43e7-a92e-fa232f28b590",
            "placement_rule": "default-placement",
            "has_instance_obj": "true",
            "quota": {
                "enabled": false,
                "max_size_kb": -1,
                "max_objects": -1
            },
            "num_shards": 0,
            "bi_shard_hash_type": 0,
            "requester_pays": "false",
            "has_website": "false",
            "swift_versioning": "false",
            "swift_ver_location": ""
        },
        "attrs": [
            {
                "key": "user.rgw.acl",
                "val": "AgJ7AAAAAwISAAAABgAAAGljdGZveAQAAABtaWtlAwNdAAAAAQEAAAAGAAAAaWN0Zm94DwAAAAEAAAAGAAAAaWN0Zm94BAMyAAAAAgIEAAAAAAAAAAYAAABpY3Rmb3gAAAAAAAAAAAICBAAAAA8AAAAEAAAAbWlrZQAAAAAAAAAA"
            },
            {
                "key": "user.rgw.idtag",
                "val": ""
            },
            {
                "key": "user.rgw.manifest",
                "val": ""
            }
        ]
    }
}

rgw的user uid信息保存格式

每个user在该pool里都有两个objects

1
2
3
# rados ls -p default.rgw.users.uid
ictfox.buckets
ictfox

查看两个objects的omap和存储数据：

1
2
3
4
# rados -p default.rgw.users.uid listomapvals ictfox
# rados -p default.rgw.users.uid get ictfox file
# cat file
ictfox   .IYZ800MDM7VF3EDLISC7(05HfpRrrh1Gs1p8bxBbcn2HcGp0n7UvuaMNPCuHSmikeictfoxIYZ800MDM7VF3EDLISC7HIYZ800MDM7VF3EDLISC7(05HfpRrrh1Gs1p8bxBbcn2HcGp0n7UvuaMNPCuHS���������������������������������
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# rados -p default.rgw.users.uid listomapkeys ictfox.buckets
bruins
# rados -p default.rgw.users.uid listomapvals ictfox.buckets
bruins
value (246 bytes) :
00000000  07 05 f0 00 00 00 00 00  00 00 0c 00 00 00 00 00  |................|
00000010  00 00 2d c7 c8 58 01 00  00 00 00 00 00 00 07 03  |..-..X..........|
00000020  c1 00 00 00 06 00 00 00  62 72 75 69 6e 73 18 00  |........bruins..|
00000030  00 00 64 65 66 61 75 6c  74 2e 72 67 77 2e 62 75  |..default.rgw.bu|
00000040  63 6b 65 74 73 2e 64 61  74 61 2c 00 00 00 30 37  |ckets.data,...07|
00000050  30 65 63 31 32 39 2d 32  65 38 34 2d 34 37 66 65  |0ec129-2e84-47fe|
00000060  2d 62 33 32 61 2d 62 30  62 66 39 39 31 34 61 35  |-b32a-b0bf9914a5|
00000070  31 66 2e 31 34 32 37 34  2e 31 2c 00 00 00 30 37  |1f.14274.1,...07|
00000080  30 65 63 31 32 39 2d 32  65 38 34 2d 34 37 66 65  |0ec129-2e84-47fe|
00000090  2d 62 33 32 61 2d 62 30  62 66 39 39 31 34 61 35  |-b32a-b0bf9914a5|
000000a0  31 66 2e 31 34 32 37 34  2e 31 19 00 00 00 64 65  |1f.14274.1....de|
000000b0  66 61 75 6c 74 2e 72 67  77 2e 62 75 63 6b 65 74  |fault.rgw.bucket|
000000c0  73 2e 69 6e 64 65 78 1a  00 00 00 64 65 66 61 75  |s.index....defau|
000000d0  6c 74 2e 72 67 77 2e 62  75 63 6b 65 74 73 2e 6e  |lt.rgw.buckets.n|
000000e0  6f 6e 2d 65 63 00 10 00  00 00 00 00 00 01 2d c7  |on-ec.........-.|
000000f0  c8 58 1a 7f b4 0b                                 |.X....|
000000f6

rgw的user keys信息保存格式

每个user在该pool里都有一个以<access_key>的值命名的object

1
2
# rados ls -p default.rgw.users.keys
IYZ800MDM7VF3EDLISC7

查看该object的omap和存储数据：

1
2
3
4
# rados -p default.rgw.users.keys listomapvals IYZ800MDM7VF3EDLISC7
# rados -p default.rgw.users.keys get IYZ800MDM7VF3EDLISC7 file
# cat file
ictfox

rgw的user bucket原数据信息保存格式

格式：.dir.<bucket id>

1
2
# rados -p default.rgw.buckets.index ls
.dir.070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1

查看该object的omap和存储数据：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# rados -p default.rgw.buckets.index get .dir.070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1 file
# cat file
#
# rados -p default.rgw.buckets.index listomapkeys .dir.070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1
hello.txt
# rados -p default.rgw.buckets.index listomapvals .dir.070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1
hello.txt
value (196 bytes) :
00000000  08 03 be 00 00 00 09 00  00 00 68 65 6c 6c 6f 2e  |..........hello.|
00000010  74 78 74 06 00 00 00 00  00 00 00 01 04 03 53 00  |txt...........S.|
00000020  00 00 01 0c 00 00 00 00  00 00 00 21 e8 c8 58 42  |...........!..XB|
00000030  2f 40 29 20 00 00 00 65  64 30 37 36 32 38 37 35  |/@) ...ed0762875|
00000040  33 32 65 38 36 33 36 35  65 38 34 31 65 39 32 62  |32e86365e841e92b|
00000050  66 63 35 30 64 38 63 06  00 00 00 69 63 74 66 6f  |fc50d8c....ictfo|
00000060  78 04 00 00 00 6d 69 6b  65 00 00 00 00 0c 00 00  |x....mike.......|
00000070  00 00 00 00 00 00 00 00  00 00 00 00 00 01 01 02  |................|
00000080  00 00 00 09 06 0a 2c 00  00 00 30 37 30 65 63 31  |......,...070ec1|
00000090  32 39 2d 32 65 38 34 2d  34 37 66 65 2d 62 33 32  |29-2e84-47fe-b32|
000000a0  61 2d 62 30 62 66 39 39  31 34 61 35 31 66 2e 31  |a-b0bf9914a51f.1|
000000b0  34 32 34 35 2e 35 00 00  00 00 00 00 00 00 00 00  |4245.5..........|
000000c0  00 00 00 00                                       |....|
000000c4

rgw的user bucket中对象数据保存格式

每个object没有omap信息

1
2
3
4
5
6
7
# rados -p default.rgw.buckets.data ls
070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1_hello.txt
# rados -p default.rgw.buckets.data get 070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1_hello.txt file
# cat file
Hello World!

# rados -p default.rgw.buckets.data listomapvals 070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1_hello.txt

查看object的xattr信息

1
2
3
4
5
6
7
8
9
10
# rados -p default.rgw.buckets.data listxattr 070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1_hello.txt
user.rgw.acl
user.rgw.cache_control
user.rgw.etag
user.rgw.idtag
user.rgw.manifest
user.rgw.pg_ver
user.rgw.source_zone
# rados -p default.rgw.buckets.data getxattr 070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1_hello.txt user.rgw.acl
{ictfoxmike]ictfoxictfox2ictfoxmike

rgw的user bucket的metadata信息

1
2
3
# rados -p default.rgw.data.root ls
bruins
.bucket.meta.bruins:070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1

每个object没有omap信息

1
2
# rados -p default.rgw.data.root listomapvals bruins
# rados -p default.rgw.data.root listomapvals .bucket.meta.bruins:070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1

查看object的存储数据

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
# rados -p default.rgw.data.root get bruins file
# cat file

�bruinsdefault.rgw.buckets.data,070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1,070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1default.rgw.buckets.indexdefault.rgw.buckets.non-ecictfoxictfox

# rados -p default.rgw.data.root get .bucket.meta.bruins:070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1 file
# ceph-dencoder type RGWBucketInfo import file decode dump_json
{
    "bucket": {
        "name": "bruins",
        "pool": "default.rgw.buckets.data",
        "data_extra_pool": "default.rgw.buckets.non-ec",
        "index_pool": "default.rgw.buckets.index",
        "marker": "070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
        "bucket_id": "070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1",
        "tenant": ""
    },
    "creation_time": "0.000000",
    "owner": "ictfox",
    "flags": 0,
    "zonegroup": "d40f14ea-4700-43e7-a92e-fa232f28b590",
    "placement_rule": "default-placement",
    "has_instance_obj": "true",
    "quota": {
        "enabled": false,
        "max_size_kb": -1,
        "max_objects": -1
    },
    "num_shards": 0,
    "bi_shard_hash_type": 0,
    "requester_pays": "false",
    "has_website": "false",
    "swift_versioning": "false",
    "swift_ver_location": ""
}

查看object的xattr

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
# rados -p default.rgw.data.root listxattr bruins
ceph.objclass.version
# rados -p default.rgw.data.root getxattr bruins ceph.objclass.version
$_osgTpZcD-jCFgcwdi3yvHmH

# rados -p default.rgw.data.root listxattr .bucket.meta.bruins:070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1
ceph.objclass.version
user.rgw.acl
# rados -p default.rgw.data.root getxattr .bucket.meta.bruins:070ec129-2e84-47fe-b32a-b0bf9914a51f.14274.1 user.rgw.acl > file
# ceph-dencoder type RGWAccessControlPolicy import file decode dump_json
{
    "acl": {
        "acl_user_map": [
            {
                "user": "ictfox",
                "acl": 15
            }
        ],
        "acl_group_map": [],
        "grant_map": [
            {
                "id": "ictfox",
                "grant": {
                    "type": {
                        "type": 0
                    },
                    "id": "ictfox",
                    "email": "",
                    "permission": {
                        "flags": 15
                    },
                    "name": "mike",
                    "group": 0
                }
            }
        ]
    },
    "owner": {
        "id": "ictfox",
        "display_name": "mike"
    }
}