集群-存储-MooseFS
MooseFS
简介:
MooseFS是一种分布式文件系统,MooseFS文件系统结构包括以下四种角色:
1 管理服务器managing server (master)
2 元数据日志服务器Metalogger server(Metalogger)
3数据存储服务器data servers (chunkservers)
4 客户机挂载使用client computers
优点: 容错;数据安全性(当一个资源分布存储在两个节点上时,一个节点down掉,可以从另一个节点读取);支持回收站功能(默认存储1天)
缺点: 由于硬件配置要求高,所以设备成本比较高
硬件条件:
双电 CPU两路 内存两组 硬盘raid1 5 10
在centos7环境,部署MooseFS文件系统:
172.16.0.160 master
172.16.0.170 logger
172.16.0.161 chunk1
172.16.0.162 chunk2
172.16.0.163 chunk3
172.16.0.181 client1
检查初始化
/etc/hosts
1. 添加用户,安装依赖的rpm包
# useradd -s /sbin/nologin mfs
# yum install -y zlib-devel make gcc gcc-c++
2. 部署master
安装
[root@master ~]# lftp 172.16.0.99
lftp 172.16.0.99:~> cd tar
cd ok, cwd=/tar
lftp 172.16.0.99:/tar> get mfs-1.6.20-2.tar.gz
[root@master ~]# scp mfs-1.6.20-2.tar.gz 172.16.0.170:/root/
[root@master ~]# scp mfs-1.6.20-2.tar.gz 172.16.0.161:/root/
[root@master ~]# scp mfs-1.6.20-2.tar.gz 172.16.0.162:/root/
[root@master ~]# scp mfs-1.6.20-2.tar.gz 172.16.0.163:/root/
[root@master ~]# scp mfs-1.6.20-2.tar.gz 172.16.0.181:/root/
[root@master ~]# tar zxf mfs-1.6.20-2.tar.gz -C /usr/local/src/
[root@master ~]# cd /usr/local/src/mfs-1.6.20-2/
[root@master /usr/local/src/mfs-1.6.20-2]# ./configure --prefix=/usr/local/mfs --with-default-user=mfs --with-default-group=mfs --disable-mfschunkserver --disable-mfsmount
--disable-mfschunkserver 不安装成chunkserver
--disable-mfsmount 不安装成client
[root@master /usr/local/src/mfs-1.6.20-2]# make
[root@master /usr/local/src/mfs-1.6.20-2]# make install
[root@master /usr/local/mfs]# ls
etc sbin share var
[root@master /usr/local/mfs/etc]# cp mfsmaster.cfg.dist mfsmaster.cfg
主配置文件 不用改
[root@master /usr/local/mfs/etc]# cp mfsexports.cfg.dist mfsexports.cfg
导出目录的配置文件 不用改
[root@master /usr/local/mfs/var/mfs]# cp metadata.mfs.empty metadata.mfs
元数据配置文件 不用改
启动master:
[root@master ~]# /usr/local/mfs/sbin/mfsmaster start
master <-> metaloggers module: listen on *:9419
master <-> chunkservers module: listen on *:9420
main master server module: listen on *:9421
[root@master ~]# echo "/usr/local/mfs/sbin/mfsmaster start" >> /etc/rc.local
[root@master ~]# chmod +x /etc/rc.d/rc.local
[root@master ~]# netstat -antp | grep mfsmaster
tcp 0 0 0.0.0.0:9419 0.0.0.0:* LISTEN 5594/mfsmaster
tcp 0 0 0.0.0.0:9420 0.0.0.0:* LISTEN 5594/mfsmaster
tcp 0 0 0.0.0.0:9421 0.0.0.0:* LISTEN 5594/mfsmaster
启动master的web监控服务:
[root@master ~]# /usr/local/mfs/sbin/mfscgiserv
[root@master ~]# echo "/usr/local/mfs/sbin/mfscgiserv " >> /etc/rc.local
测试
浏览器访问:http://172.16.0.160:9425
2. 配置logger
安装并编辑配置文件
[root@logger ~]# tar zxf mfs-1.6.20-2.tar.gz -C /usr/local/src/
[root@logger ~]# cd /usr/local/src/mfs-1.6.20-2/
[root@logger /usr/local/src/mfs-1.6.20-2]# ./configure --prefix=/usr/local/mfs --with-default-user=mfs --with-default-group=mfs --disable-mfschunkserver --disable-mfsmount
[root@logger /usr/local/src/mfs-1.6.20-2]# make
[root@logger /usr/local/src/mfs-1.6.20-2]# make install
[root@logger /usr/local/mfs]# ls
etc sbin share var
[root@logger /usr/local/mfs]# cd etc/
[root@logger /usr/local/mfs/etc]# ls
mfsexports.cfg.dist mfsmaster.cfg.dist mfsmetalogger.cfg.dist
[root@logger /usr/local/mfs/etc]# cp mfsmetalogger.cfg.dist mfsmetalogger.cfg
主配置文件
[root@logger /usr/local/mfs/etc]# vim mfsmetalogger.cfg
14 MASTER_HOST = master
[root@logger /usr/local/mfs/etc]# ping master
PING master.up.com (172.16.0.160) 56(84) bytes of data.
64 bytes from master.up.com (172.16.0.160): icmp_seq=1 ttl=64 time=0.416 ms
启动服务:
[root@logger ~]# /usr/local/mfs/sbin/mfsmetalogger start
[root@logger ~]# echo "/usr/local/mfs/sbin/mfsmetalogger start" >> /etc/rc.local
[root@logger ~]# chmod +x /etc/rc.d/rc.local
[root@logger ~]# netstat -antp | grep logger
tcp 0 0 172.16.0.170:49156 172.16.0.160:9419 ESTABLISHED 5563/mfsmetalogger
3. 配置chunkserver 1 2 3
(1)准备存储
chunk1
[root@chunk1 ~]# pvcreate /dev/sdb
[root@chunk1 ~]# vgcreate vgmfs /dev/sdb
[root@chunk1 ~]# lvcreate -L 10G -n lvmfs vgmfs
[root@chunk1 ~]# mkfs.xfs /dev/vgmfs/lvmfs
[root@chunk1 ~]# mkdir /data
[root@chunk1 ~]# vim /etc/fstab
/dev/mapper/vgmfs-lvmfs /data xfs defaults 0 0
[root@chunk1 ~]# mount -a
[root@chunk1 ~]# df -h | tail -1
/dev/mapper/vgmfs-lvmfs 10G 33M 10G 1% /data
chunk2
[root@chunk2 ~]# pvcreate /dev/sdb
[root@chunk2 ~]# vgcreate vgmfs /dev/sdb
[root@chunk2 ~]# lvcreate -L 10G -n lvmfs vgmfs
[root@chunk2 ~]# mkfs.xfs /dev/vgmfs/lvmfs
[root@chunk2 ~]# mkdir /data
[root@chunk2 ~]# vim /etc/fstab
/dev/mapper/vgmfs-lvmfs /data xfs defaults 0 0
[root@chunk2 ~]# mount -a
[root@chunk2 ~]# df -h | tail -1
/dev/mapper/vgmfs-lvmfs 10G 33M 10G 1% /data
chunk3
[root@chunk3 ~]# pvcreate /dev/sdb
[root@chunk3 ~]# vgcreate vgmfs /dev/sdb
[root@chunk3 ~]# lvcreate -L 10G -n lvmfs vgmfs
[root@chunk3 ~]# mkfs.xfs /dev/vgmfs/lvmfs
[root@chunk3 ~]# mkdir /data
[root@chunk3 ~]# vim /etc/fstab
/dev/mapper/vgmfs-lvmfs /data xfs defaults 0 0
[root@chunk3 ~]# mount -a
[root@chunk3 ~]# df -h | tail -1
/dev/mapper/vgmfs-lvmfs 10G 33M 10G 1% /data
(2)安装mfs
[root@chunk1 ~]# tar zxf mfs-1.6.20-2.tar.gz -C /usr/local/src/
[root@chunk1 ~]# cd /usr/local/src/mfs-1.6.20-2/
[root@chunk1 /usr/local/src/mfs-1.6.20-2]# ./configure --prefix=/usr/local/mfs --with-default-user=mfs --with-default-group=mfs --disable-mfsmaster --disable-mfsmount
[root@chunk1 /usr/local/src/mfs-1.6.20-2]# make
[root@chunk1 /usr/local/src/mfs-1.6.20-2]# make install
[root@chunk2 ~]# tar zxf mfs-1.6.20-2.tar.gz -C /usr/local/src/
[root@chunk2 ~]# cd /usr/local/src/mfs-1.6.20-2/
[root@chunk2 /usr/local/src/mfs-1.6.20-2]# ./configure --prefix=/usr/local/mfs --with-default-user=mfs --with-default-group=mfs --disable-mfsmaster --disable-mfsmount
[root@chunk2 /usr/local/src/mfs-1.6.20-2]# make
[root@chunk2 /usr/local/src/mfs-1.6.20-2]# make install
[root@chunk3 ~]# tar zxf mfs-1.6.20-2.tar.gz -C /usr/local/src/
[root@chunk3 ~]# cd /usr/local/src/mfs-1.6.20-2/
[root@chunk3 /usr/local/src/mfs-1.6.20-2]# ./configure --prefix=/usr/local/mfs --with-default-user=mfs --with-default-group=mfs --disable-mfsmaster --disable-mfsmount
[root@chunk3 /usr/local/src/mfs-1.6.20-2]# make
[root@chunk3 /usr/local/src/mfs-1.6.20-2]# make install
[root@chunk1 /usr/local/mfs]# ls
etc sbin share var
[root@chunk2 /usr/local/mfs]# ls
etc sbin share var
[root@chunk3 /usr/local/mfs]# ls
etc sbin share var
主配置文件
[root@chunk1 /usr/local/mfs/etc]# cp mfschunkserver.cfg.dist mfschunkserver.cfg
[root@chunk1 /usr/local/mfs/etc]# vim mfschunkserver.cfg
12 MASTER_HOST = master
[root@chunk2 /usr/local/mfs/etc]# cp mfschunkserver.cfg.dist mfschunkserver.cfg
[root@chunk2 /usr/local/mfs/etc]# vim mfschunkserver.cfg
12 MASTER_HOST = master
[root@chunk3 /usr/local/mfs/etc]# cp mfschunkserver.cfg.dist mfschunkserver.cfg
[root@chunk3 /usr/local/mfs/etc]# vim mfschunkserver.cfg
12 MASTER_HOST = master
存储配置文件:
[root@chunk1 /usr/local/mfs/etc]# cp mfshdd.cfg.dist mfshdd.cfg
[root@chunk1 /usr/local/mfs/etc]# vim mfshdd.cfg
/data ##lv的挂载点
[root@chunk2 /usr/local/mfs/etc]# cp mfshdd.cfg.dist mfshdd.cfg
[root@chunk2 /usr/local/mfs/etc]# vim mfshdd.cfg
/data ##lv的挂载点
[root@chunk3 /usr/local/mfs/etc]# cp mfshdd.cfg.dist mfshdd.cfg
[root@chunk3 /usr/local/mfs/etc]# vim mfshdd.cfg
/data ##lv的挂载点
[root@chunk1 ~]# chown -R mfs:mfs /data/
[root@chunk2 ~]# chown -R mfs:mfs /data/
[root@chunk3 ~]# chown -R mfs:mfs /data/
启动服务:
[root@chunk1 ~]# /usr/local/mfs/sbin/mfschunkserver start
[root@chunk2 ~]# /usr/local/mfs/sbin/mfschunkserver start
[root@chunk3 ~]# /usr/local/mfs/sbin/mfschunkserver start
[root@chunk1 ~]# echo "/usr/local/mfs/sbin/mfschunkserver start" >> /etc/rc.local
[root@chunk1 ~]# chmod +x /etc/rc.d/rc.local
[root@chunk2 ~]# echo "/usr/local/mfs/sbin/mfschunkserver start" >> /etc/rc.local
[root@chunk2 ~]# chmod +x /etc/rc.d/rc.local
[root@chunk3 ~]# echo "/usr/local/mfs/sbin/mfschunkserver start" >> /etc/rc.local
[root@chunk3 ~]# chmod +x /etc/rc.d/rc.local
[root@chunk1 ~]# netstat -antp | grep mfs
tcp 0 0 0.0.0.0:9422 0.0.0.0:* LISTEN 5567/mfschunkserver
tcp 0 0 172.16.0.161:53894 172.16.0.160:9420 ESTABLISHED 5567/mfschunkserve
[root@chunk2 ~]# netstat -antp | grep mfs
tcp 0 0 0.0.0.0:9422 0.0.0.0:* LISTEN 5611/mfschunkserver
tcp 0 0 172.16.0.162:60774 172.16.0.160:9420 ESTABLISHED 5611/mfschunkserver
[root@chunk3 ~]# netstat -antp | grep mfs
tcp 0 0 0.0.0.0:9422 0.0.0.0:* LISTEN 5600/mfschunkserver
tcp 0 0 172.16.0.163:58348 172.16.0.160:9420 ESTABLISHED 5600/mfschunkserver
4. 配置client
[root@client1 ~]# useradd -s /sbin/nologin mfs
[root@client1 ~]# yum install -y zlib-devel make gcc gcc-c++ fuse fuse-libs fuse-devel
[root@client1 ~]# tar zxf mfs-1.6.20-2.tar.gz -C /usr/local/src/
[root@client1 ~]# cd /usr/local/src/mfs-1.6.20-2/
[root@client1 /usr/local/src/mfs-1.6.20-2]# ./configure --prefix=/usr/local/mfs --with-default-user=mfs --with-default-group=mfs --disable-mfsmaster --disable-mfschunkserver
[root@client1 /usr/local/src/mfs-1.6.20-2]# make
[root@client1 /usr/local/src/mfs-1.6.20-2]# make install
[root@client1 /usr/local/mfs]# ls
bin etc sbin share
5. 使用mfs存储
[root@client1 ~]# vim /etc/pfofile
PATH=/usr/local/mfs/bin:$PATH
[root@client1 ~]# . /etc/profile
[root@client1 ~]# echo $PATH
/usr/local/mfs/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/root/bin
[root@client1 ~]# mfstools -h
tools:
mfsgetgoal 查看文件的副本数,默认是1个
mfssetgoal 设定......
mfsgettrashtime 查看回收站的保存时间,默认1天
mfssettrashtime 设置
mfscheckfile 检查文件
mfsfileinfo 查看文件信息
mfsappendchunks
mfsdirinfo 查看目录信息
mfsfilerepair 修复文件
mfsmakesnapshot 创建快照
mfsgeteattr 查看权限
mfsseteattr 设置权限
mfsdeleattr 删除权限
deprecated tools: 递归设置
mfsrgetgoal = mfsgetgoal -r
mfsrsetgoal = mfssetgoal -r
mfsrgettrashtime = mfsgettreshtime -r
mfsrsettrashtime = mfssettreshtime -r
[root@client1 ~]# mkdir /mnt/mfs
[root@client1 ~]# mfsmount /mnt/mfs/ -H 172.16.0.160
挂载
[root@client1 ~]# df -h | tail -1
172.16.0.160:9421 30G 0 30G 0% /mnt/mfs
测试使用:
[root@client1 /mnt/mfs]# mkdir dir1 dir2
[root@client1 /mnt/mfs]# mfssetgoal -r 2 dir2
将dir2下的文件,保存为2个副本
[root@client1 /mnt/mfs]# cp /etc/passwd dir1/
[root@client1 /mnt/mfs]# cp /etc/passwd dir2/
[root@client1 /mnt/mfs]# mfsfileinfo dir1/passwd
dir1/passwd:
chunk 0: 0000000000000001_00000001 / (id:1 ver:1)
copy 1: 172.16.0.161:9422
[root@client1 /mnt/mfs]# mfsfileinfo dir2/passwd
dir2/passwd:
chunk 0: 0000000000000002_00000001 / (id:2 ver:1)
copy 1: 172.16.0.162:9422
copy 2: 172.16.0.163:9422
关掉 161 162
结果:
dir1/下的文件不可用
dir2/下的文件可用
[root@client1 /mnt/mfs]# rm -f dir1/passwd
[root@client1 /mnt/mfs]# mfsgettrashtime dir1
dir1: 86400
使用回收站:
[root@client1 /mnt]# mkdir meta
[root@client1 /mnt]# mfsmount -m /mnt/meta/ -H 172.16.0.160
[root@client1 /mnt/meta]# cd trash/
[root@client1 /mnt/meta/trash]# ls
00000004|dir1|passwd undel
[root@client1 /mnt/meta/trash]# mv 00000004\|dir1\|passwd undel/
维护:
1. 关闭
(1)client卸载使用
卸载之前挂载过的目录
(2)停止chunkserver
[root@chunk1 ~]# /usr/local/mfs/sbin/mfschunkserver -s
(3)停止logger
[root@logger ~]# /usr/local/mfs/sbin/mfsmetalogger -s
(4)停止master
[root@master ~]# /usr/local/mfs/sbin/mfsmaster -s
2. 开启,设置了开机自启动
(1)master
(2)logger
(3)chunkserver
(4)client
MFS瓶颈就是master
两台master可以搭建HA 集中存储
mfs一些有用的东西
[root@master /usr/local/mfs/var/mfs]# ls