首页 > 代码库 > 44 分布式存储与mogilefs、mogiles使用进阶
44 分布式存储与mogilefs、mogiles使用进阶
01 分布式存储与mogilefs
实战：安装配置mogilefs
配置环境：
node1：192.168.1.130 CentOS7.2
node2：192.168.1.131 CentOS7.2
node3：192.168.1.132 CentOS7.2
node4：192.168.1.133 CentOS7.2
[root@node1 ~]# vim /etc/hosts
添加
192.168.1.130 node1
192.168.1.131 node2
192.168.1.132 node3
192.168.1.133 node4
[root@node1 ~]# scp /etc/hosts node2:/etc
[root@node1 ~]# scp /etc/hosts node3:/etc
[root@node1 ~]# scp /etc/hosts node4:/etc
[root@node2 ~]# rpm -ivh epel-release-latest-7.noarch.rpm
[root@node3 ~]# rpm -ivh epel-release-latest-7.noarch.rpm
[root@node4 ~]# rpm -ivh epel-release-latest-7.noarch.rpm
[root@node2 ~]# ls *rpm
MogileFS-Server-2.46-2.el6.noarch.rpm
MogileFS-Server-mogilefsd-2.46-2.el6.noarch.rpm
MogileFS-Server-mogstored-2.46-2.el6.noarch.rpm
MogileFS-Utils-2.19-1.el6.noarch.rpm
perl-Danga-Socket-1.61-1.el6.rf.noarch.rpm
perl-MogileFS-Client-1.14-1.el6.noarch.rpm
perl-Perlbal-1.78-1.el6.noarch.rpm
[root@node2 ~]# yum install perl-Sys-Syslog perl-Net-Netmask perl-IO-AIO -y
[root@node2 ~]# yum install *rpm -y
[root@node2 ~]# scp *rpm node3:/root
[root@node2 ~]# scp *rpm node4:/root
[root@node3 ~]# yum install perl-Sys-Syslog perl-Net-Netmask perl-IO-AIO -y
[root@node3 ~]# yum install *rpm -y
[root@node4 ~]# yum install perl-Sys-Syslog perl-Net-Netmask perl-IO-AIO -y
[root@node4 ~]# yum install *rpm -y
[root@node2 ~]# yum -y install mariadb-server
[root@node2 ~]# vim /etc/my.cnf
添加：
innodb_file_per_table = 1
skip_name_resolve = 1
[root@node2 ~]# systemctl start mariadb.service
[root@node2 ~]# systemctl enable mariadb.service
[root@node2 ~]# mysql
MariaDB [(none)]> GRANT ALL ON *.* TO ‘root‘@‘192.168.1.%‘ IDENTIFIED BY ‘mageedu‘;
MariaDB [(none)]> CREATE DATABASE mogilefs;
MariaDB [(none)]> GRANT ALL ON mogilefs.* TO ‘moguser‘@‘192.168.1.%‘ IDENTIFIED BY ‘mogpass‘;
MariaDB [(none)]> FLUSH PRIVILEGES;
MariaDB [(none)]> \q
[root@node2 ~]# mogdbsetup --dbhost=192.168.1.131 --dbrootpass=‘mageedu‘ --dbuser=‘moguser‘ --dbpass=‘mogpass‘
[root@node2 ~]# mysql
MariaDB [(none)]> use mogilefs;
MariaDB [mogilefs]> SHOW TABLES;
+----------------------+
| Tables_in_mogilefs |
+----------------------+
| checksum |
| class |
| device |
| domain |
| file |
| file_on |
| file_on_corrupt |
| file_to_delete |
| file_to_delete2 |
| file_to_delete_later |
| file_to_queue |
| file_to_replicate |
| fsck_log |
| host |
| server_settings |
| tempfile |
| unreachable_fids |
+----------------------+
[root@node2 ~]# cd /etc/mogilefs/
#Tracker节点配置
[root@node2 mogilefs]# vim mogilefsd.conf
修改
db_dsn = DBI:mysql:mogilefs:host=127.0.0.1
db_user = username
db_pass = password
为
db_dsn = DBI:mysql:mogilefs:host=192.168.1.131
db_user = moguser
db_pass = mogpass
修改
listen = 127.0.0.1:7001
为
listen = 0.0.0.0:7001
[root@node2 mogilefs]# service mogilefsd start
#storage节点（可以是所有节点）
[root@node2 mogilefs]# mkdir -p /data/mogilefs/dev1
[root@node2 mogilefs]# chown -R mogilefs.mogilefs /data/mogilefs/
[root@node2 mogilefs]# vim /etc/mogilefs/mogstored.conf
修改
docroot = /var/mogdata
为
docroot = /data/mogilefs/
[root@node2 mogilefs]# vim /etc/init.d/mogstored
在lockfile=（15行左右）添加
pidfile=/var/run/mogilefsd/mogstored.pid
修改
[ $RETVAL = 0 ] && success && touch ${lockfile} || failure #约28行
为
[ $RETVAL = 0 ] && success && touch ${lockfile} && echo $(pidof mogstored) > ${pidfile} || failure
修改
[ $RETVAL = 0 ] && success && rm -f ${lockfile} || failure #约37行
为
[ $RETVAL = 0 ] && success && rm -f ${lockfile} ${pidfile} || failure
[root@node2 ~]# service mogstored start
[root@node3 ~]# mkdir -p /data/mogilefs/dev2
[root@node3 ~]# chown -R mogilefs.mogilefs /data/mogilefs/
[root@node4 ~]# mkdir -p /data/mogilefs/dev3
[root@node4 ~]# chown -R mogilefs.mogilefs /data/mogilefs/
[root@node2 mogilefs]# scp /etc/rc.d/init.d/mogstored node3:/etc/rc.d/init.d/
[root@node2 mogilefs]# scp /etc/rc.d/init.d/mogstored node4:/etc/rc.d/init.d/
[root@node2 mogilefs]# scp /etc/mogilefs/mogstored.conf node3:/etc/mogilefs/
[root@node2 mogilefs]# scp /etc/mogilefs/mogstored.conf node4:/etc/mogilefs/
[root@node3 ~]# systemctl start mogstored.service
[root@node3 ~]# systemctl daemon-reload
[root@node3 ~]# service mogstored restart
[root@node4 ~]# systemctl daemon-reload
[root@node4 ~]# service mogstored start
02 mogiles使用进阶
mogilefs基本用法
#检测节点
[root@node2 mogilefs]# mogadm check
Checking trackers...
127.0.0.1:7001 ... OK
Checking hosts...
No devices found on tracker(s).
[root@node2 mogilefs]# mogadm --trackers=192.168.1.131:7001 check
Checking trackers...
192.168.1.131:7001 ... OK
Checking hosts...
No devices found on tracker(s).
#添加节点
[root@node2 ~]# mogadm --trackers=192.168.1.131:7001 host add 192.168.1.131 --ip=192.168.1.131 --status=alive
[root@node2 ~]# mogadm host list
192.168.1.131 [1]: alive
IP: 192.168.1.131:7500
[root@node2 ~]# mogadm --trackers=192.168.1.131:7001 host add 192.168.1.132 --ip=192.168.1.132 --status=alive
[root@node2 ~]# mogadm --trackers=192.168.1.131:7001 host add 192.168.1.133 --ip=192.168.1.133 --status=alive
[root@node2 ~]# mogadm host list
192.168.1.131 [1]: alive
IP: 192.168.1.131:7500
192.168.1.132 [2]: alive
IP: 192.168.1.132:7500
192.168.1.133 [3]: alive
IP: 192.168.1.133:7500
#更改节点状态
[root@node2 ~]# mogadm host mark 192.168.1.132 down
[root@node2 ~]# mogadm host list
192.168.1.131 [1]: alive
IP: 192.168.1.131:7500
192.168.1.132 [2]: down
IP: 192.168.1.132:7500
192.168.1.133 [3]: alive
IP: 192.168.1.133:7500
[root@node2 ~]# mogadm host mark 192.168.1.132 alive
[root@node2 ~]# mogadm host list
192.168.1.131 [1]: alive
IP: 192.168.1.131:7500
192.168.1.132 [2]: alive
IP: 192.168.1.132:7500
192.168.1.133 [3]: alive
IP: 192.168.1.133:7500
#查看设备
[root@node2 ~]# mogadm device list
192.168.1.131 [1]: alive
used(G) free(G) total(G) weight(%)
192.168.1.132 [2]: alive
used(G) free(G) total(G) weight(%)
192.168.1.133 [3]: alive
used(G) free(G) total(G) weight(%)
#添加设备
[root@node2 ~]# mogadm device add 192.168.1.131 1
[root@node2 ~]# mogadm device list
192.168.1.131 [1]: alive
used(G) free(G) total(G) weight(%)
dev1: alive 4.375 15.615 19.990 100
192.168.1.132 [2]: alive
used(G) free(G) total(G) weight(%)
192.168.1.133 [3]: alive
used(G) free(G) total(G) weight(%)
[root@node2 ~]# mogadm device add 192.168.1.132 2
[root@node2 ~]# mogadm device add 192.168.1.133 3
[root@node2 ~]# mogadm device list
192.168.1.131 [1]: alive
used(G) free(G) total(G) weight(%)
dev1: alive 4.375 15.615 19.990 100
192.168.1.132 [2]: alive
used(G) free(G) total(G) weight(%)
dev2: alive 4.246 15.744 19.990 100
192.168.1.133 [3]: alive
used(G) free(G) total(G) weight(%)
dev3: alive 4.246 15.744 19.990 100
[root@node2 ~]# mogadm check
Checking trackers...
127.0.0.1:7001 ... OK
Checking hosts...
[ 1] 192.168.1.131 ... OK
[ 2] 192.168.1.132 ... OK
[ 3] 192.168.1.133 ... OK
Checking devices...
host device size(G) used(G) free(G) use% ob state I/O%
---- ------------ ---------- ---------- ---------- ------ ---------- -----
[ 1] dev1 19.990 4.376 15.614 21.89% writeable 0.0
[ 2] dev2 19.990 4.246 15.744 21.24% writeable 0.0
[ 3] dev3 19.990 4.246 15.744 21.24% writeable 0.0
---- ------------ ---------- ---------- ---------- ------
total: 59.971 12.869 47.102 21.46%
[root@node2 ~]# mogadm domain add images
[root@node2 ~]# mogadm domain add files
[root@node2 ~]# mogadm domain list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
images default 2 MultipleHosts() NONE
[root@node2 ~]# mogadm class list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
images default 2 MultipleHosts() NONE
[root@node2 ~]# mogadm class add images jpeg
[root@node2 ~]# mogadm class add images png
[root@node2 ~]# mogadm class add images gif
[root@node2 ~]# mogadm class list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
images default 2 MultipleHosts() NONE
images gif 2 MultipleHosts() NONE
images jpeg 2 MultipleHosts() NONE
images png 2 MultipleHosts() NONE
[root@node2 ~]# mogadm class add files plaintext --mindevcount=1
[root@node2 ~]# mogadm class list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
files plaintext 1 MultipleHosts() NONE
images default 2 MultipleHosts() NONE
images gif 2 MultipleHosts() NONE
images jpeg 2 MultipleHosts() NONE
images png 2 MultipleHosts() NONE
[root@node2 ~]# mogadm class add files html --replpolicy="MultipleHosts(3)"
[root@node2 ~]# mogadm class list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
files html 2 MultipleHosts(3) NONE
files plaintext 1 MultipleHosts() NONE
images default 2 MultipleHosts() NONE
images gif 2 MultipleHosts() NONE
images jpeg 2 MultipleHosts() NONE
images png 2 MultipleHosts() NONE
#上传文件
[root@node2 ~]# mogupload --trackers=192.168.1.131:7001 --domain=files --key=‘/fstab.txt‘ --file=‘/etc/fstab‘ --class=plaintext
[root@node2 ~]# mogfileinfo --trackers=192.168.1.131:7001 --domain=files --key=‘/fstab.txt‘- file: /fstab.txt
class: plaintext
devcount: 1
domain: files
fid: 2
key: /fstab.txt
length: 465
- http://192.168.1.131:7500/dev1/0/000/000/0000000002.fid
#可以通过上面网址访问文件
[root@node2 ~]# mogupload --trackers=192.168.1.131:7001 --domain=images --class=jpeg --key=‘/linux.jpg‘ --file=‘./01.jpg‘
[root@node2 ~]# mogfileinfo --trackers=192.168.1.131:7001 --domain=images --key=‘/linux.jpg‘- file: /linux.jpg
class: jpeg
devcount: 1
domain: images
fid: 3
key: /linux.jpg
length: 60582
- http://192.168.1.132:7500/dev2/0/000/000/0000000003.fid
[root@node2 ~]# mogupload --trackers=192.168.1.131:7001 --domain=images --key=‘/linux2.jpg‘ --file=‘./02.jpg‘
[root@node2 ~]# mogfileinfo --trackers=192.168.1.131:7001 --domain=images --key=‘/linux2.jpg‘
- file: /linux2.jpg
class: default
devcount: 1
domain: images
fid: 4
key: /linux2.jpg
length: 116463
- http://192.168.1.132:7500/dev2/0/000/000/0000000004.fid
#查看上传的文件
[root@node2 ~]# moglistkeys --trackers=192.168.1.131:7001 --domain=images
/linux.jpg
/linux2.jpg
[root@node2 ~]# moglistkeys --trackers=192.168.1.131:7001 --domain=files
/fstab.txt
[root@node2 ~]# moglistfids --trackers=192.168.1.131:7001 --domain=files
fid 2
class plaintext
devcount 1
domain files
key /fstab.txt
length 465
fid 3
class jpeg
devcount 1
domain images
key /linux.jpg
length 60582
fid 4
class default
devcount 1
domain images
key /linux2.jpg
length 116463
[root@node2 ~]# mogstats --db_dsn="DBI:mysql:mogilefs:host=192.168.1.131" --db_user="moguser" --db_pass="mogpass" --verbose --stats="devices,files"
Fetching statistics... (devices,files)
... files stats...
... done
... per-device stats...
... done
Statistics for devices...
device host files status
---------- ---------------- ------------ ----------
dev1 192.168.1.131 1 alive
dev2 192.168.1.132 2 alive
---------- ---------------- ------------ ----------
Statistics for files...
domain class files size (m) fullsize (m)
-------------------- ----------- ---------- ----------- -------------
files plaintext 1 0 0
images default 1 0 0
images jpeg 1 0 0
-------------------- ----------- ---------- ----------- -------------
done
[root@node2 ~]# mogstats --db_dsn="DBI:mysql:mogilefs:host=192.168.1.131" --db_user="moguser" --db_pass="mogpass" --verbose --stats="all"
Fetching statistics... (all)
... replication stats...
... done
... replication queue stats...
... done
... delete queue stats...
... done
... files stats...
... done
... per-device stats...
... done
... fid stats...
... done
... queue stats...
... done
Statistics for devices...
device host files status
---------- ---------------- ------------ ----------
dev1 192.168.1.131 1 alive
dev2 192.168.1.132 2 alive
---------- ---------------- ------------ ----------
Statistics for file ids...
Max file id: 4
Statistics for files...
domain class files size (m) fullsize (m)
-------------------- ----------- ---------- ----------- -------------
files plaintext 1 0 0
images default 1 0 0
images jpeg 1 0 0
-------------------- ----------- ---------- ----------- -------------
Statistics for replication...
domain class devcount files
-------------------- ----------- ---------- ----------
files plaintext 1 1
images default 1 1
images jpeg 1 1
-------------------- ----------- ---------- ----------
Statistics for replication queue...
status count
-------------------- ------------
deferred 2
-------------------- ------------
Statistics for delete queue...
status count
-------------------- ------------
-------------------- ------------
Statistics for general queues...
queue status count
--------------- -------------------- ------------
--------------- -------------------- ------------
done
#删除文件
[root@node2 ~]# mogdelete --trackers=192.168.1.131:7001 --domain=images --key=‘/linux.jpg‘
[root@node2 ~]# mogdelete --trackers=192.168.1.131:7001 --domain=images --key=‘/linux2.jpg‘
[root@node2 ~]# mogdelete --trackers=192.168.1.131:7001 --domain=files --key=‘/fstab.txt‘
#
[root@node2 ~]# mogadm class delete images jpeg
[root@node2 ~]# mogadm class delete images png
[root@node2 ~]# mogadm class delete images gif
[root@node2 ~]# mogadm class delete files plaintext
[root@node2 ~]# mogadm class delete files html
[root@node2 ~]# mogadm class list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
files default 2 MultipleHosts() NONE
images default 2 MultipleHosts() NONE
[root@node2 ~]# mogadm domain delete files
[root@node2 ~]# mogadm domain delete images
[root@node2 ~]# mogadm domain add imgs
[root@node2 ~]# mogadm domain list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
imgs default 2 MultipleHosts() NONE
[root@node2 ~]# mogupload --trackers=192.168.1.131:7001 --domain=imgs --key=‘linux.jpg‘ --file=‘./01.jpg‘
[root@node2 ~]# mogfileinfo --trackers=192.168.1.131:7001 --domain=imgs --key=‘linux.jpg‘
- file: linux.jpg
class: default
devcount: 1
domain: imgs
fid: 5
key: linux.jpg
length: 60582
- http://192.168.1.131:7500/dev1/0/000/000/0000000005.fid
[root@node2 ~]# mogadm class add imgs jpeg --mindevcount=2 --replpolicy="MultipleHosts(2)"[root@node2 ~]# mogadm class list
domain class mindevcount replpolicy hashtype
-------------------- -------------------- ------------- ------------ -------
imgs default 2 MultipleHosts() NONE
imgs jpeg 2 MultipleHosts(2) NONE
[root@node2 ~]# mogupload --trackers=192.168.1.131:7001 --domain=imgs --class=jpeg --key=‘linux2.jpg‘ --file=‘./02.jpg‘
[root@node2 ~]# mogfileinfo --trackers=192.168.1.131:7001 --domain=imgs --key=‘linux2.jpg‘- file: linux2.jpg
class: jpeg
devcount: 1
domain: imgs
fid: 6
key: linux2.jpg
length: 116463
- http://192.168.1.131:7500/dev1/0/000/000/0000000006.fid
实战：Nginx做为MogileFS的前端客户端
一、安装Nginx：
1、解决依赖关系
[root@node1 ~]# yum groupinstall "Development Tools" "Server Platform Deveopment"
[root@node1 ~]# yum -y install openssl-devel pcre-devel
2、安装
首先添加用户nginx，实现以之运行nginx服务进程：
[root@node1 ~]# groupadd -r nginx
[root@node1 ~]# useradd -r -g nginx nginx
[root@node1 ~]# tar xf nginx_mogilefs_module-1.0.4.tar.gz
[root@node1 ~]# tar xf tengine-2.0.1.tar.gz
[root@node1 ~]# cd tengine-2.0.1/
[root@node1 tengine-2.0.1]# ./configure \
--prefix=/usr \
--sbin-path=/usr/sbin/nginx \
--conf-path=/etc/nginx/nginx.conf \
--error-log-path=/var/log/nginx/error.log \
--http-log-path=/var/log/nginx/access.log \
--pid-path=/var/run/nginx/nginx.pid \
--lock-path=/var/lock/nginx.lock \
--user=nginx \
--group=nginx \
--with-http_ssl_module \
--with-http_flv_module \
--with-http_stub_status_module \
--with-http_gzip_static_module \
--http-client-body-temp-path=/var/tmp/nginx/client/ \
--http-proxy-temp-path=/var/tmp/nginx/proxy/ \
--http-fastcgi-temp-path=/var/tmp/nginx/fcgi/ \
--http-uwsgi-temp-path=/var/tmp/nginx/uwsgi \
--http-scgi-temp-path=/var/tmp/nginx/scgi \
--with-pcre \
--with-debug \
--add-module=../nginx_mogilefs_module-1.0.4
[root@node1 tengine-2.0.1]# vim objs/Makefile
修改
CFLAGS = -pipe -O -W -Wall -Wpointer-arith -Wno-unused-parameter -Werror -g
为
CFLAGS = -pipe -O -W -Wall -Wpointer-arith -Wno-unused-parameter -g
[root@node1 tengine-2.0.1]# make && make install
[root@node1 tengine-2.0.1]# vim /usr/lib/systemd/system/nginx.service
[Unit]
Description=The nginx HTTP and reverse proxy server
After=network.target remote-fs.target nss-lookup.target
[Service]
Type=forking
PIDFile=/var/run/nginx/nginx.pid
ExecStartPre=/usr/sbin/nginx -t
ExecStart=/usr/sbin/nginx
ExecReload=/bin/kill -s HUP $MAINPID
KillMode=process
KillSignal=SIGQUIT
TimeoutStopSec=5
PrivateTmp=true
[Install]
WantedBy=multi-user.target
[root@node1 tengine-2.0.1]# systemctl daemon-reload
[root@node1 tengine-2.0.1]# systemctl start nginx.service
#测试不成功
[root@node1 ~]# cd /etc/nginx/
[root@node1 nginx]# cp nginx.conf{,.bak}
[root@node1 nginx]# vim nginx.conf
添加
location /images/ {
mogilefs_tracker 192.168.1.131:7001;
mogilefs_domain imgs;
mogilefs_methods GET;
mogilefs_noverify on;
mogilefs_pass {
proxy_pass $mogilefs_path;
proxy_hide_header Content-Type;
proxy_buffering off;
}
}
[root@node1 nginx]# nginx -t
[root@node1 nginx]# systemctl restart nginx.service
[root@node2 ~]# moglistkeys --trackers=192.168.1.131:7001 --domain=imgs
linux.jpg
linux2.jpg
[root@node2 ~]# mogdelete --trackers=192.168.1.131:7001 --domain=imgs --key=‘linux.jpg‘
[root@node2 ~]# mogdelete --trackers=192.168.1.131:7001 --domain=imgs --key=‘linux2.jpg‘
[root@node2 ~]# mogupload --trackers=192.168.1.131:7001 --domain=imgs --key=‘linux.jpg‘ --file=‘./01.jpg‘
[root@node2 ~]# systemctl restart nginx.service
[root@node2 ~]# mogupload --trackers=192.168.1.131:7001 --domain=imgs --key=‘linux2.jpg‘ --file=‘./02.jpg‘
[root@node2 ~]# scp /etc/mogilefs/mogilefsd.conf node3:/etc/mogilefs/
[root@node2 ~]# scp /etc/mogilefs/mogilefsd.conf node4:/etc/mogilefs/
[root@node3 ~]# systemctl start mogilefsd.service
[root@node2 ~]# moglistkeys --trackers=192.168.1.132:7001 --domain=imgs
linux.jpg
linux2.jpg
[root@node4 ~]# systemctl start mogilefsd.service
[root@node2 ~]# moglistkeys --trackers=192.168.1.133:7001 --domain=imgs
linux.jpg
linux2.jpg
[root@node1 ~]# vim /etc/nginx/nginx.conf
在38行左右添加
upstream trackers {
server 192.168.1.131:7001;
server 192.168.1.132:7001;
server 192.168.1.133:7001;
check interval=1000 rise=2 fall=5 timeout=1000;
}
修改location为
location /images/ {
mogilefs_tracker trackers;
mogilefs_domain imgs;
mogilefs_methods GET;
mogilefs_noverify on;
mogilefs_pass {
proxy_pass $mogilefs_path;
proxy_hide_header Content-Type;
proxy_buffering off;
}
}
location /status {
check_status;
}
[root@node1 ~]# systemctl restart nginx.service
64:43(58848)
本文出自 “追梦” 博客，请务必保留此出处http://sihua.blog.51cto.com/377227/1878311
44 分布式存储与mogilefs、mogiles使用进阶