Tomcat+Nginx+Memcached集羣部署

主機環境 redhat6.5 64位

實驗環境 服務端1 ip172.25.29.1   nginx

服務端2 ip 172.25.29.2    tomcat+memcached

服務端3 ip 172.25.29.3    tomcat+memcached

安裝包  jdk-7u79-linux-x64.tar.gz

 apache-tomcat-7.0.37.tar.gz

 nginx-goodies-nginx-sticky-module-ng-c78b7dd79d0d.tar.gz

 asm-3.2.jar

 kryo-1.04.jar

 kryo-serializers-0.10.jar

 memcached-session-manager-1.6.3.jar

 memcached-session-manager-tc7-1.6.3.jar

 minlog-1.2.jar

 msm-kryo-serializer-1.6.3.jar

 reflectasm-1.01.jar

 spymemcached-2.7.3.jar

防火牆狀態 關閉

 

1.jdk安裝、環境配置及測試服務端2

1.解壓、做軟連接

[root@server2mnt]# tar zxf jdk-7u79-linux-x64.tar.gz -C /usr/local/  #解壓jdk

[root@server2mnt]# cd /usr/local/      #切換到解壓目錄

[root@server2local]# ls

bin  etc games  include  jdk1.7.0_79 lib  lib64  libexec sbin  share  src

[root@server2local]# ln -s jdk1.7.0_79/ java      #做軟連接

[root@server2local]# ll

total44

drwxr-xr-x.2 root root 4096 Jun 28  2011 bin

drwxr-xr-x.2 root root 4096 Jun 28  2011 etc

drwxr-xr-x.2 root root 4096 Jun 28  2011 games

drwxr-xr-x.2 root root 4096 Jun 28  2011 include

lrwxrwxrwx.1 root root   12 Sep 24 10:50 java ->jdk1.7.0_79/    #查看

drwxr-xr-x.8 uucp  143 4096 Apr 11  2015 jdk1.7.0_79

drwxr-xr-x.2 root root 4096 Jun 28  2011 lib

drwxr-xr-x.2 root root 4096 Jun 28  2011 lib64

drwxr-xr-x.2 root root 4096 Jun 28  2011 libexec

drwxr-xr-x.2 root root 4096 Jun 28  2011 sbin

drwxr-xr-x.5 root root 4096 Aug  8 21:38 share

drwxr-xr-x.2 root root 4096 Jun 28  2011 src

2.將jdk添加到環境變量

[root@server2local]# vim /etc/profile   

 79 export JAVA_HOME=/usr/local/java

 80 exportCLASSPATH=.:$JAVA_HOME/lib:$JAVA_HOME/jre/lib

 81 export PATH=$PATH:$JAVA_HOME/bin

[root@server2local]# source /etc/profile

3.測試

[root@server2local]# echo $JAVA_HOME    #測試添加環境變量是否成功

/usr/local/java

[root@server2local]# echo $CLASSPATH

.:/usr/local/java/lib:/usr/local/java/jre/lib

[root@server2local]# echo $JAVA_HOME     

/usr/local/java

[root@server2local]# cd java       #測試java的環境是否配置成功

[root@server2java]# vim test.java    #寫個簡單的測試頁

  1 public class test{

  2        public static void main(String[] args)

  3        {

  4                 System.out.println("Helloworld!");

  5        }

  6 }

[root@server2java]# javac test.java    #連接生成test.class文件

[root@server2mnt]# java test    #執行

Helloworld!

 

2.tomcat的安裝服務端2

1.解壓、做軟連接

[root@server2mnt]# tar zxf apache-tomcat-7.0.8.tar.gz -C /usr/local/  #解壓

[root@server2mnt]# cd /usr/local/

[root@server2local]# ls

apache-tomcat-7.0.8  etc   include  jdk1.7.0_79  lib64   sbin   src

bin                  games  java    lib          libexec  share

[root@server2local]# ln -s apache-tomcat-7.0.8/ tomcat   #做軟連接

[root@server2local]# ll

total48

drwxr-xr-x.9 root root 4096 Sep 24 11:11 apache-tomcat-7.0.8

drwxr-xr-x.2 root root 4096 Jun 28  2011 bin

drwxr-xr-x.2 root root 4096 Jun 28  2011 etc

drwxr-xr-x.2 root root 4096 Jun 28  2011 games

drwxr-xr-x.2 root root 4096 Jun 28  2011 include

lrwxrwxrwx.1 root root   12 Sep 24 10:50 java ->jdk1.7.0_79/

drwxr-xr-x.8 uucp  143 4096 Sep 24 11:02 jdk1.7.0_79

drwxr-xr-x.2 root root 4096 Jun 28  2011 lib

drwxr-xr-x.2 root root 4096 Jun 28  2011 lib64

drwxr-xr-x.2 root root 4096 Jun 28  2011 libexec

drwxr-xr-x.2 root root 4096 Jun 28  2011 sbin

drwxr-xr-x.5 root root 4096 Aug  8 21:38 share

drwxr-xr-x.2 root root 4096 Jun 28  2011 src

lrwxrwxrwx.1 root root   20 Sep 24 11:12 tomcat-> apache-tomcat-7.0.8/     #查看

[root@server2local]# cd tomcat/bin

[root@server2bin]# ./startup.sh     #開啓tomcat

 

2.測試

[root@server2bin]#cd ..

[root@server2tomcat]# vim webapps/ROOT/test.jsp    #寫測試儀頁

  1 server2-The Time is <%=newjava.util.Date()%>

#測試 172.25.29.2:8080

172.25.29.2:8080/test.jsp

在服務端3上進行一樣的配置也能夠用scp把服務端2上java和tomcat目錄傳過去以下

[root@server2local]# scp -r java/ tomcat/ 172.25.29.3:/usr/local/

在進行系統環境里加上java的配置啓動tomcat便可

 

3.Nginx添加sticky  (服務端1)

1.nginx負載均衡已經配置好了

在前面的博客裏已經寫過nginx源碼安裝這裏就再也不重複了也能夠參考前面的博客

[root@server1~]# cd /usr/local/lnmp/nginx/conf  

[root@server1conf]# vim nginx.conf

 20    upstream wen {

 21                 server 172.25.29.2:8080;    #輪詢機制

 22                 server 172.25.29.3:8080;

 23        }

 49        location / {

 50            root   html;

 51            index  index.html index.jspindex.php index.htm;   #默認發佈目錄

 52        }

 

 68        location ~ \.jsp$ {

 69            proxy_pass   http://wen;

 70        }

[root@server1conf]# nginx -t    #檢測

nginx:the configuration file /usr/local/lnmp/nginx/conf/nginx.conf syntax is ok

nginx:configuration file /usr/local/lnmp/nginx/conf/nginx.conf test is successful

[root@server1conf]# nginx -s reload    #刷新

測試 172.25.29.1

 

172.25.29.1/test.jsp

刷新以後

也就是server2 和server3相互交替

 

2.改變負載均衡機制爲sticky

因爲nginx是輪詢機制若是在訪問頁面如填寫信息時忽然卡住刷新以後就會跳到另外一個server上就得從新開始填寫。可是Nginx裏有不少算法其中的ip_hash也能夠防止這些問題。使用ip_hash有一個問題是當客戶端和服務器之間使用cdn內容分發系統高速緩存時客戶端的訪問到達cdn由cdn訪問服務器識別的ip是cdn的ip那麼將集中於訪問後臺的一臺服務器(x至關於DDOS***)會加速服務器的損壞。那麼爲了防止這些問題就有用下面的算法sticky不是nginx裏自帶的nginx-sticky-module爲 nginx 的第三方模塊,使 nginx 支持 sticky 模式,因此須要將包加入配置、從新編譯、安裝nginx

 

1.從新源碼安裝nginx添加一個模塊

[root@server1local]# nginx -s stop      #關閉nginx

[root@server1mnt]# tar zxf nginx-goodies-nginx-sticky-module-ng-c78b7dd79d0d.tar.gz #解壓

[root@server1mnt]# ls

nginx-1.8.1.tar.gz

nginx-goodies-nginx-sticky-module-ng-c78b7dd79d0d

nginx-goodies-nginx-sticky-module-ng-c78b7dd79d0d.tar.gz

 

[root@server1nginx-1.8.1]# make clean    #清除上一次的緩存文件

rm-rf Makefile objs

[root@server1nginx-1.10.1]# ./configure --prefix=/usr/local/lnmp/nginx   #從新配置--with-http_ssl_module--with-http_stub_status_module--add-module=/mnt/nginx-goodies-nginx-sticky-module-ng-c78b7dd79d0d

[root@server1nginx-1.8.1]# make     #編譯、連接

[root@server1nginx-1.8.1]# make install   #安裝

[root@server1nginx-1.8.1]#cd /usr/local/lnmp/nginx/conf

[root@server1conf]# vim nginx.conf

 18        upstream westos{

 19                 sticky;     #使用sticky

 20                 server 172.25.29.2:8080;

 21                 server 172.25.29.3:8080;

[root@server1local]# nginx -t   #檢測nginx文件裏是否有錯誤

nginx:the configuration file /usr/local/lnmp/nginx/conf/nginx.conf syntax is ok

nginx:configuration file /usr/local/lnmp/nginx/conf/nginx.conf test is successful

[root@server1local]# nginx   #啓動

2.測試

測試 172.25.29.1/test.jsp

刷新以後結果不變仍是

 

4交叉存儲避免單點故障添加memcached服務端2

配置好以後若是一臺服務器宕機了那麼這臺服務器正在運行的業務將直接結束正在存儲的數據丟失。爲了防止這些問題Tomcat1將session存儲到Tomcat2的memcached中當Tomcat2的memcached不可用時Tomcat1將session存儲到本身的memcached上Tomcat2則正好相反。使用這種配置就避免了單點故障。

1.安裝、開啓memcached、寫測試頁

[root@server2ROOT]# yum install -y memcached      #安裝memcached

[root@server2ROOT]# /etc/init.d/memcached start      #開啓memcached

Startingmemcached:                                        [  OK  ]

[root@server2tomcat]# bin/shutdown.sh   #關閉Tomcat

[root@server2ROOT]# vim test.jsp    #寫一個jsp的測試頁面

  1 <%@ page contentType="text/html;charset=GBK" %>

  2 <%@ page import="java.util.*"%>

  3<html><head><title>Cluster AppTest</title></head>

  4 <body>

  5 Server Info:

  6 <%

  7 out.println(request.getLocalAddr() + ": " + request.getLocalPort()+"<br>");%    >

  8 <%

  9 out.println("<br> ID " + session.getId()+"<br>");

 10 String dataName =request.getParameter("dataName");

 11 if (dataName != null &&dataName.length() > 0) {

 12 String dataValue =request.getParameter("dataValue");

 13 session.setAttribute(dataName, dataValue);

 14 }

 15 out.print("<b>Sessionlist</b>");

 16 Enumeration e =session.getAttributeNames();

 17 while (e.hasMoreElements()) {

 18 String name = (String)e.nextElement();

 19 String value =session.getAttribute(name).toString();

 20 out.println( name + " = " +value+"<br>");

 21 System.out.println( name + " = "+ value);

 22 }

 23 %>

 24 <form action="test.jsp"method="POST">

 25 name:<input type=text size=20name="dataName">

 26 <br>

 27 key:<input type=text size=20name="dataValue">

 28 <br>

 29 <input type=submit>

 30 </form>

 31 </body>

 32 </html>

[root@server2ROOT]# cd /usr/local/tomcat/lib

[root@server2mnt]# ls /mnt/update/      #先前下載好的包

asm-3.2.jar                              minlog-1.2.jar

kryo-1.04.jar                           msm-kryo-serializer-1.6.3.jar

kryo-serializers-0.10.jar                reflectasm-1.01.jar

memcached-session-manager-1.6.3.jar      spymemcached-2.7.3.jar

memcached-session-manager-tc7-1.6.3.jar

[root@server2mnt]# cd /usr/local/tomcat/lib/

[root@server2lib]# mv /mnt/update/* .    #將包移動到指定路徑

[root@server2lib]# ls

annotations-api.jar                 memcached-session-manager-tc7-1.6.3.jar

asm-3.2.jar                          minlog-1.2.jar

catalina-ant.jar                    msm-kryo-serializer-1.6.3.jar

catalina-ha.jar                      reflectasm-1.01.jar

catalina.jar                         servlet-api.jar

catalina-tribes.jar                  spymemcached-2.7.3.jar

ecj-4.2.1.jar                        tomcat-api.jar

el-api.jar                           tomcat-coyote.jar

jasper-el.jar                        tomcat-dbcp.jar

jasper.jar                           tomcat-i18n-es.jar

jsp-api.jar                          tomcat-i18n-fr.jar

kryo-1.04.jar                        tomcat-i18n-ja.jar

kryo-serializers-0.10.jar            tomcat-jdbc.jar

memcached-session-manager-1.6.3.jar  tomcat-util.jar

[root@server2lib]# cd ..

UsingCATALINA_BASE:   /usr/local/tomcat

UsingCATALINA_HOME:   /usr/local/tomcat

UsingCATALINA_TMPDIR: /usr/local/tomcat/temp

UsingJRE_HOME:        /usr/local/java

UsingCLASSPATH:       /usr/local/tomcat/bin/bootstrap.jar:/usr/local/tomcat/bin/tomcat-juli.jar

[root@server2tomcat]# vim conf/context.xml    #添加節點及節點失效後該怎麼存儲

 34 <ManagerclassName="de.javakaffee.web.msm.MemcachedBackupSessionManager"

 35memcachedNodes="n1:172.25.29.2:11211,n2:172.25.29.3:11211"     #結點

 36 failoverNodes="n1"           #當n2失效時存儲到n1上

 37requestUriIgnorePattern=".*\.(ico|png|gif|jpg|css|js)$"

 38transcoderFactoryClass="de.javakaffee.web.msm.serializer.kryo.KryoTranscoderF    actory"

 39 />

[root@server2tomcat]# bin/startup.sh   #開啓Tomcat

服務端3和服務端2的配置基本相同惟一不一樣的是上面提到的/usr/local/tomcat/conf/context.xml裏的36行把「n1」改爲「n2」能夠用scp傳過去再作修改。

 

2.測試

測試 172.25.29.1/test.jsp

添加用戶

添加完成

添加3個用戶

                              

[root@server2tomcat]# tail -f logs/catalina.out      #查看日誌

Sep28, 2016 11:39:53 PM de.javakaffee.web.msm.serializer.kryo.KryoTranscoder<init>

INFO:Starting with initialBufferSize 102400 and maxBufferSize 2048000

Sep28, 2016 11:39:53 PM de.javakaffee.web.msm.MemcachedSessionServicestartInternal

INFO:MemcachedSessionService finished initialization, sticky true, operation timeout1000, with node ids [n2] and failover node ids [n1]

Sep28, 2016 11:39:53 PM org.apache.coyote.AbstractProtocol start

INFO:Starting ProtocolHandler ["http-bio-8080"]

Sep28, 2016 11:39:53 PM org.apache.coyote.AbstractProtocol start

INFO:Starting ProtocolHandler ["ajp-bio-8009"]

Sep28, 2016 11:39:53 PM org.apache.catalina.startup.Catalina start

INFO:Server startup in 1921 ms

user1= 111

user2= 222

user1= 111           #上面建立的用戶

user2= 222

user1= 111

user3= 333

^C

[root@server2tomcat]# telnet 172.25.29.3 11211  #遠程登錄服務端3查看是不是否寫進入11211是memcached的端口號

Trying172.25.29.3...

Connectedto 172.25.29.3.

Escapecharacter is '^]'.

get50B9BAB1CBB21C9BF884CC3613560752-n2   #get後面的是上面截圖裏的ID

VALUE50B9BAB1CBB21C9BF884CC3613560752-n2 2048 125

[1]WWqt2Wqt01WqtWqt#50B9BAB1CBB21C9BF884CC3613560752-n2

user2

222user1

111user3

333       #大概能夠看出來添加的用戶

END

quit

Connectionclosed by foreign host.

 

[root@server2tomcat]# bin/shutdown.sh  #關閉Tomcat1

UsingCATALINA_BASE:   /usr/local/tomcat

UsingCATALINA_HOME:   /usr/local/tomcat

UsingCATALINA_TMPDIR: /usr/local/tomcat/temp

UsingJRE_HOME:        /usr/local/java

UsingCLASSPATH:      /usr/local/tomcat/bin/bootstrap.jar:/usr/local/tomcat/bin/tomcat-juli.jar

 

當關閉Tomcat1續輸入數據時並無中斷而是跳到Tomcat2上且數據並未丟失

[root@server2tomcat]# telnet 172.25.29.3 11211     #遠程登錄服務端3

Trying172.25.29.3...

Connectedto 172.25.29.3.

Escapecharacter is '^]'.

get50B9BAB1CBB21C9BF884CC3613560752-n2

VALUE50B9BAB1CBB21C9BF884CC3613560752-n2 2048 137

[1]WWqt2Wqu01WquWqu4#50B9BAB1CBB21C9BF884CC3613560752-n2

---

user2  222user1  111user4  444user3  333   #大概能夠看到數據仍是寫入服務端3

END

quit

Connectionclosed by foreign host.