分佈式文件系統的實現

分佈式文件系統的實現

 

1 配置數據庫節點node5(172.16.21.5)

 

Mysql的cmake編譯與安裝，詳細步步驟參照個人"Mysql的cmake編譯與安裝博文"

[root@node5 ~]# mysql

MariaDB [(none)]> create database mogilefs;

Query OK, 1 row affected (0.02 sec)

 

MariaDB [(none)]> use mogilefs

Database changed

#受權root用戶全部權限

MariaDB [mogilefs]> grant all on *.* to 'root'@'172.16.21.5';

Query OK, 0 rows affected (0.00 sec)

 

 

#受權mogilefs的tracker鏈接數據庫的用戶

MariaDB [mogilefs]> grant all on *.* to 'moguser'@'172.16.%.%' identified by 'mogpass' with grant option;

Query OK, 0 rows affected (0.00 sec)

 

MariaDB [mogilefs]> flush privileges;

Query OK, 0 rows affected (0.00 sec)

 

二、安裝配置mogilefs（三臺主機: node2:172.16.21.2, node3:172.16.21.3 ,node4:172.16.21.4）

（1）安裝

 

 

 

 

 

[root@node2 ~]# ls

anaconda-ks.cfg copyvbox.sh install.log.syslog mariadb-10.0.13.tar.gz mysql.sh

centos6.6.repo install.log mariadb-10.0.13 mogilefs percona-toolkit-2.2.4-1.noarch.rpm

[root@node2 ~]# cd mogilefs/

[root@node2 mogilefs]# ls

MogileFS-Server-2.46-2.el6.noarch.rpm Perlbal-doc-1.78-1.el6.noarch.rpm

MogileFS-Server-mogilefsd-2.46-2.el6.noarch.rpm perl-MogileFS-Client-1.14-1.el6.noarch.rpm

MogileFS-Server-mogstored-2.46-2.el6.noarch.rpm perl-Net-Netmask-1.9015-8.el6.noarch.rpm

MogileFS-Utils-2.19-1.el6.noarch.rpm perl-Perlbal-1.78-1.el6.noarch.rpm

Perlbal-1.78-1.el6.noarch.rpm

[root@node2 mogilefs]# yum -y install *.rpm perl-IO-AIO

 

（2）配置tracker

節點node2(172.16.21.2)

   ①、修改mogilefsd的配置文件

[root@node2 ~]#vim /etc/mogilefs/mogilefsd.conf

# Enable daemon mode to work in background and use syslog

daemonize = 1

# Where to store the pid of the daemon (must be the same in the init script)

pidfile = /var/run/mogilefsd/mogilefsd.pid

# Database connection information

db_dsn = DBI:mysql:mogilefs:host=172.16.21.5   #數據庫

db_user = moguser        #數據庫受權的用戶

db_pass = mogpass        #用戶密碼

# IP:PORT to listen on for mogilefs client requests

listen = 172.16.21.2:7001    #監聽的端口及IP

# Optional, if you don't define the port above.

conf_port = 7001

# Number of query workers to start by default.

query_jobs = 10

# Number of delete workers to start by default.

delete_jobs = 1

# Number of replicate workers to start by default.

replicate_jobs = 5

# Number of reaper workers to start by default.

# (you don't usually need to increase this)

reaper_jobs = 1

 

 

 

②、爲mogilefsd設定數據庫

[root@node2 ~]# mogdbsetup --dbhost=172.16.21.5 --dbname=mogilefs --dbrootuser=root --dbuser=moguser --dbpass=mogpass

 

This will attempt to setup or upgrade your MogileFS database.

It won't destroy existing data.

Run with --help for more information. Run with --yes to shut up these prompts.

 

Continue? [N/y]: y

 

[root@node2 ~]#

 

[root@node2 ~]# service mogilefsd start

Starting mogilefsd [ OK ]

 

③、查看7001端口是否成功監聽

[root@node2 ~]# ss –tnlp

 

 

 

 

 

 

 

 

 

 

 

 ④、驗證數據庫(在節點node5:172.16.21.5)

 

 

[root@node5 ~]# mysql

MariaDB [(none)]> use mogilefs

Database changed

MariaDB [mogilefs]> show tables;

+----------------------+

| Tables_in_mogilefs |

+----------------------+

| checksum |

| class |

| device |

| domain |

| file |

| file_on |

| file_on_corrupt |

| file_to_delete |

| file_to_delete2 |

| file_to_delete_later |

| file_to_queue |

| file_to_replicate |

| fsck_log |

| host |

| server_settings |

| tempfile |

| unreachable_fids |

+----------------------+

17 rows in set (0.00 sec)

 

MariaDB [mogilefs]>

 

 

說明我在節點二上的設置的mogilefs已經開啓

 

爲了完整性，接下來繼續在節點node2上操做：

建立一個磁盤分區：

fdisk /dev/sda

n p 3 +30G w

 

[root@node2 ~]# partx -a /dev/sda

磁盤初始化

[root@node2 ~]# mke2fs -t ext4 /dev/sda3

 

 

 

[root@node2 ~]# mkdir /mogilefs

接下來就是把磁盤/dev/sda3掛載 到 /mogilefs

 

[root@node2 ~]# vim /etc/fstab

/dev/sda3 /mogilefs ext4 defaults 0 0

 

[root@node2 ~]# mount –a

 

[root@node2 ~]# mount

 

 

[root@node2 ~]# chown -R mogilefs.mogilefs /mogilefs/

 

#########配置mogstored#########

 

[root@node2 ~]# vim /etc/mogilefs/mogstored.conf

 

maxconns = 10000

httplisten = 0.0.0.0:7500

mgmtlisten = 0.0.0.0:7501

docroot = /mogilefs

 

 

（4）查看7500端口監聽狀態

 

[root@node2 ~]# service mogstored restart

 

 

 

 

（5）添加主機

 

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 host add 172.16.21.2 --ip=172.16.21.2 --status=alive

###查看添加主機命令

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 host list

172.16.21.2 [1]: alive

IP: 172.16.21.2:7500

 

[root@node2 ~]#

（6）添加設備

 

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 device add 172.16.21.2 1

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 device list

172.16.21.2 [1]: alive

used(G) free(G) total(G) weight(%)

dev1: alive 0.042 27.866 27.908 100

 

[root@node2 ~]#

 

（7）添加domain

 

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 domain add images

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 domain add files

###列出domain###

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 domain list

domain class mindevcount replpolicy hashtype

-------------------- -------------------- ------------- ------------ -------

files default 2 MultipleHosts() NONE

 

images default 2 MultipleHosts() NONE

 

 

（8）添加class

 

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 class add images class1

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 class add images class2

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 class list

domain class mindevcount replpolicy hashtype

-------------------- -------------------- ------------- ------------ -------

files default 2 MultipleHosts() NONE

 

images class1 2 MultipleHosts() NONE

images class2 2 MultipleHosts() NONE

images default 2 MultipleHosts() NONE

 

 

 

 

（9）配置其它兩臺主機的mogilefsd和mogstored

 

由於這三臺主機的tracker區用一個數據庫，因此在安裝配置時，不需再從新設定數據庫，只配置相應的mogilefsd和mogstored配置文件就能夠了

 

（10）在節點node2（172.16.21.2）查看最後的狀態信息

①、列出全部主機

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 host list

172.16.21.2 [1]: alive

IP: 172.16.21.2:7500

 

172.16.21.3 [2]: alive

IP: 172.16.21.3:7500

 

172.16.21.4 [3]: alive

IP: 172.16.21.4:7500

 

 

 ②、列出全部設備

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 device list

172.16.21.2 [1]: alive

used(G) free(G) total(G) weight(%)

dev1: alive 0.043 27.865 27.908 100

 

172.16.21.3 [2]: alive

used(G) free(G) total(G) weight(%)

dev2: alive 0.043 27.865 27.908 100

 

172.16.21.4 [3]: alive

used(G) free(G) total(G) weight(%)

dev3: alive 0.043 27.865 27.908 100

 

 

 

 

 

 

 

 

③、列出全部domain

 

[root@node2 ~]# mogadm --trackers=172.16.21.2:7001 domain list

domain class mindevcount replpolicy hashtype

-------------------- -------------------- ------------- ------------ -------

files default 2 MultipleHosts() NONE

 

images class1 2 MultipleHosts() NONE

images class2 2 MultipleHosts() NONE

images class3 2 MultipleHosts() NONE

images class4 2 MultipleHosts() NONE

images default 2 MultipleHosts() NONE

 

music default 2 MultipleHosts() NONE

 

note default 2 MultipleHosts() NONE

 

 

 

 

（11）上傳圖片資源到mogilefs

 

[root@node2 ~]# mogupload --trackers=172.16.21.2:7001 --domain=images --key='1.jpg' --file='/root/pictures/1.jpg'

 

 

（12）使用mogfileinfo獲取上傳圖片的訪問路徑

 

[root@node2 ~]# moglistkeys --trackers=172.16.21.2:7001 --domain=images

1.jpg

 

 

 

[root@node2 ~]# mogfileinfo --trackers=172.16.21.2:7001 --domain=images --key='1.jpg'

- file: 1.jpg

class: default

devcount: 2

domain: images

fid: 5

key: 1.jpg

length: 104372

- http://172.16.21.2:7500/dev1/0/000/000/0000000005.fid

- http://172.16.21.3:7500/dev2/0/000/000/0000000005.fid

 

 

 

5、配置nginx服務器(172.16.21.1)

 

 

一、解壓nginx_mogilefs_module-1.0.4.zip模塊

 

[root@localhost ~]# unzip nginx_mogilefs_module-1.0.4.zip

 

lftp 172.16.0.1:/pub/Sources/sources/nginx> get nginx-1.6.2.tar.gz

 

[root@localhost ~]# tar xf nginx-1.6.2.tar.gz

 

 

 

1、安裝Nginx：

 

一、解決依賴關係

 

# yum groupinstall "Development Tools" "Server Platform Deveopment"

# yum install openssl-devel pcre-devel

 

二、安裝

 

首先添加用戶nginx，實現以之運行nginx服務進程：

# groupadd -r nginx

# useradd -r -g nginx nginx

 

3開始編譯：

[root@localhost ~]#cd nginx-1.6.2

接着開始編譯和安裝：

# ./configure \

--prefix=/usr \

--sbin-path=/usr/sbin/nginx \

--conf-path=/etc/nginx/nginx.conf \

--error-log-path=/var/log/nginx/error.log \

--http-log-path=/var/log/nginx/access.log \

--pid-path=/var/run/nginx/nginx.pid \

--lock-path=/var/lock/nginx.lock \

--user=nginx \

--group=nginx \

--with-http_ssl_module \

--with-http_flv_module \

--with-http_stub_status_module \

--with-http_gzip_static_module \

--http-client-body-temp-path=/var/tmp/nginx/client/ \

--http-proxy-temp-path=/var/tmp/nginx/proxy/ \

--http-fastcgi-temp-path=/var/tmp/nginx/fcgi/ \

--http-uwsgi-temp-path=/var/tmp/nginx/uwsgi \

--http-scgi-temp-path=/var/tmp/nginx/scgi \

--with-pcre \

--with-debug \

--add-module=/root/nginx_mogilefs_module-1.0.4

# make && make install

 

 

 

4.爲nginx提供SysV init腳本:

 

新建文件/etc/rc.d/init.d/nginx，內容以下：

#!/bin/sh

#

# nginx - this script starts and stops the nginx daemon

#

# chkconfig: - 85 15

# description: Nginx is an HTTP(S) server, HTTP(S) reverse \

# proxy and IMAP/POP3 proxy server

# processname: nginx

# config: /etc/nginx/nginx.conf

# config: /etc/sysconfig/nginx

# pidfile: /var/run/nginx.pid

 

# Source function library.

. /etc/rc.d/init.d/functions

 

# Source networking configuration.

. /etc/sysconfig/network

 

# Check that networking is up.

[ "$NETWORKING" = "no" ] && exit 0

 

nginx="/usr/sbin/nginx"

prog=$(basename $nginx)

 

NGINX_CONF_FILE="/etc/nginx/nginx.conf"

 

[ -f /etc/sysconfig/nginx ] && . /etc/sysconfig/nginx

 

lockfile=/var/lock/subsys/nginx

 

make_dirs() {

# make required directories

user=`nginx -V 2>&1 | grep "configure arguments:" | sed 's/[^*]*--user=\([^ ]*\).*/\1/g' -`

options=`$nginx -V 2>&1 | grep 'configure arguments:'`

for opt in $options; do

if [ `echo $opt | grep '.*-temp-path'` ]; then

value=`echo $opt | cut -d "=" -f 2`

if [ ! -d "$value" ]; then

# echo "creating" $value

mkdir -p $value && chown -R $user $value

fi

fi

done

}

 

start() {

[ -x $nginx ] || exit 5

[ -f $NGINX_CONF_FILE ] || exit 6

make_dirs

echo -n $"Starting $prog: "

daemon $nginx -c $NGINX_CONF_FILE

retval=$?

echo

[ $retval -eq 0 ] && touch $lockfile

return $retval

}

 

stop() {

echo -n $"Stopping $prog: "

killproc $prog -QUIT

retval=$?

echo

[ $retval -eq 0 ] && rm -f $lockfile

return $retval

}

 

restart() {

configtest || return $?

stop

sleep 1

start

}

 

reload() {

configtest || return $?

echo -n $"Reloading $prog: "

killproc $nginx -HUP

RETVAL=$?

echo

}

 

force_reload() {

restart

}

 

configtest() {

$nginx -t -c $NGINX_CONF_FILE

}

 

rh_status() {

status $prog

}

 

rh_status_q() {

rh_status >/dev/null 2>&1

}

 

case "$1" in

start)

rh_status_q && exit 0

$1

;;

stop)

rh_status_q || exit 0

$1

;;

restart|configtest)

$1

;;

reload)

rh_status_q || exit 7

$1

;;

force-reload)

force_reload

;;

status)

rh_status

;;

condrestart|try-restart)

rh_status_q || exit 0

;;

*)

echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload|configtest}"

exit 2

esac

 

 

 

5.然後爲此腳本賦予執行權限：

# chmod +x /etc/rc.d/init.d/nginx

 

添加至服務管理列表，並讓其開機自動啓動：

# chkconfig --add nginx

# chkconfig nginx on

 

6.然後就能夠啓動服務並測試了：

# service nginx start

 

 

 

七、修改nginx配置文件

只需在location段中增長以下幾行代碼

location / {

mogilefs_tracker 172.16.21.3:7001;

mogilefs_domain images;

mogilefs_methods get;

 

mogilefs_pass {

proxy_pass $mogilefs_path;

proxy_hide_header Content-Type;

proxy_buffering off;

}

}

 

 

 

 

 

使用curl上傳文件。

curl -X PUT -T '/etc/passwd' http://172.16.100.15/upload/passwd.html