Nginx + varnish 構建高可用CDN節點集群
一、 環境描述
Linux server A (CentOS release 5.8 Final) 實IP:192.168.4.97 虛IP：192.168.4.96
Linux server B (CentOS release 5.8 Final) 實IP:192.168.4.99 虛IP：192.168.4.98
域名環境（DNS輪詢解析到虛IP）：
server.osapub.com 192.168.4.96
server.osapub.com 192.168.4.98
二、 簡單架構示意圖
描述：前端兩臺NGINX做DNS輪詢，通過虛IP漂移+HA監控腳本相結合，實現前端兩臺NGINX高可用，利用NGINX反向代理功能對后端varnish實現高可用集群。
三、軟件環境搭建
3.1 編譯安裝nginx
?
#!/bin/bash
####nginx 環境安裝腳本，注意環境不同可能導致腳本運行出錯，如果環境不同建議手工一條一條執行指令。
#創建工作目錄
mkdir -p /dist/{dist,src}
cd /dist/dist
#下載安裝包
wget http://bbs.osapub.com/down/google-perftools-1.8.3.tar.gz &> /dev/null
wget http://bbs.osapub.com/down/libunwind-0.99.tar.gz &> /dev/null
wget http://bbs.osapub.com/down/pcre-8.01.tar.gz &> /dev/null
wget http://bbs.osapub.com/down/nginx-1.0.5.tar.gz &> /dev/null
#------------------------------------------------------------------------
# 使用Google的開源TCMalloc庫，憂化性能
cd /dist/src
tar zxf ../dist/libunwind-0.99.tar.gz
cd libunwind-0.99/
## 注意這里不能加其它 CFLAGS加速編譯參數
CFLAGS=-fPIC ./configure
make clean
make CFLAGS=-fPIC
make CFLAGS=-fPIC install
if [ "$?" == "0" ]; then
echo "libunwind-0.99安裝成功." >> ./install_log.txt
else
echo "libunwind-0.99安裝失敗." >> ./install_log.txt
exit 1
fi
##----------------------------------------------------------
## 使用Google的開源TCMalloc庫，提高MySQL在高并發情況下的性能
cd /dist/src
tar zxf ../dist/google-perftools-1.8.3.tar.gz
cd google-perftools-1.8.3/
CHOST="x86_64-pc-linux-gnu" CFLAGS="-march=nocona -O2 -pipe" CXXFLAGS="-march=nocona -O2 -pipe" \
./configure
make clean
make && make install
if [ "$?" == "0" ]; then
echo "google-perftools-1.8.3安裝成功." >> ./install_log.txt
else
echo "google-perftools-1.8.3安裝失敗." >> ./install_log.txt
exit 1
fi
echo "/usr/local/lib" > /etc/ld.so.conf.d/usr_local_lib.conf
/sbin/ldconfig
################ 安裝nginx ##########################
#安裝Nginx所需的pcre庫
cd /dist/src
tar zxvf ../dist/pcre-8.01.tar.gz
cd pcre-8.01/
CHOST="x86_64-pc-linux-gnu" CFLAGS="-march=nocona -O2 -pipe" CXXFLAGS="-march=nocona -O2 -pipe" \
./configure
make && make install
if [ "$?" == "0" ]; then
echo "pcre-8.01安裝成功." >> ./install_log.txt
else
echo "pcre-8.01安裝失敗." >> ./install_log.txt
exit 1
fi
cd ../
## 安裝Nginx
## 為優化性能，可以安裝 google 的 tcmalloc ，之前己經安裝過了
## 所以我們編譯 Nginx 時，加上參數 --with-google_perftools_module
## 然后在啟動nginx前需要設置環境變量 export LD_PRELOAD=/usr/local/lib/libtcmalloc.so
## 加上 -O2 參數也能優化一些性能
##
## 默認的Nginx編譯選項里居然是用 debug模式的(-g參數)，在 auto/cc/gcc 文件最底下，去掉那個 -g 參數
## 就是將 CFLAGS="$CFLAGS -g" 修改為 CFLAGS="$CFLAGS" 或者直接刪除這一行
cd /dist/src
rm -rf nginx-1.0.5
tar zxf ../dist/nginx-1.0.5.tar.gz
cd nginx-1.0.5/
sed -i 's#CFLAGS="$CFLAGS -g"#CFLAGS="$CFLAGS "#' auto/cc/gcc
make clean
CHOST="x86_64-pc-linux-gnu" CFLAGS="-march=nocona -O2 -pipe" CXXFLAGS="-march=nocona -O2 -pipe" \
./configure --user=www --group=www \
--prefix=/usr/local/nginx \
--with-http_stub_status_module \
--with-google_perftools_module
make && make install
if [ "$?" == "0" ]; then
echo "nginx-1.0.5安裝成功." >> ./install_log.txt
else
echo "nginx-1.0.5安裝失敗." >> ./install_log.txt
exit 1
fi
cd ../
#創建Nginx日志目錄
mkdir -p /data/logs
chmod +w /data/logs
chown -R www:www /data/logs
cd /usr/local/nginx/
mv conf conf_bak
ln -s /data/conf/nginx/ conf
echo 'export LD_PRELOAD=/usr/local/lib/libtcmalloc.so' > /root/nginx_start
echo 'ulimit -SHn 51200' >> /root/nginx_start
echo '/usr/local/nginx/sbin/nginx -c /usr/local/nginx/conf/nginx.conf' >> /root/nginx_start
echo '/usr/local/nginx/sbin/nginx -t' > /root/nginx_reload
echo 'kill -HUP `cat /usr/local/nginx/logs/nginx.pid`' >> /root/nginx_reload
chmod 700 /root/nginx_* 3.2 編譯安裝varnish
#!/bin/bash #進入工作目錄 cd /dist/dist #下載安裝包 wget <a href="\"http://bbs.osapub.com/down/varnish-3.0.0.tar.gz\"" target="\"_blank\"">http://bbs.osapub.com/down/varnish-3.0.0.tar.gz</a> &> /dev/null cd /dist/src rm -fr varnish-3.0.0 export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig tar zxvf ../dist/varnish-3.0.0.tar.gz cd varnish-3.0.0 #編譯參數可以根據自己需要定制 ./configure -prefix=/usr/local/varnish -enable-debugging-symbols -enable-developer-warnings -enable-dependency-tracking make && make install if [ "$?" == "0" ]; then echo "varnish-3.0.0安裝成功." >> ./install_log.txt else echo "varnish-3.0.0安裝失敗." >> ./install_log.txt exit 1 fi #設置啟動、重啟腳本 cat > /root/varnish_restart.sh <<EOF #!/bin/bash pkill varnish pkill varnish ulimit -SHn 51200 /usr/local/varnish/sbin/varnishd -n /data/varnish -f /usr/local/varnish/vcl.conf -a 0.0.0.0:81 -s malloc,12G -g www -u www -w 4000,12000,10 -T 127.0.0.1:3500 -p http_max_hdr=256 -p http_req_hdr_len=8192 EOF cat > /root/varnish_start.sh <<EOF #!/bin/bash ulimit -SHn 51200 /usr/local/varnish/sbin/varnishd -n /data/varnish -f /usr/local/varnish/vcl.conf -a 0.0.0.0:81 -s malloc,12G -g www -u www -w 4000,12000,10 -T 127.0.0.1:3500 -p http_max_hdr=256 -p http_req_hdr_len=8192 EOF chmod 755 /root/varnish*
三、 配置varnish一些參數說明：
Backend servers
Varnish有后端（或稱為源）服務器的概念。后端服務器是指Varnish提供加速服務的那臺，通常提供內容。
第一件要做的事情是告訴Varnish，哪里能找到要加速的內容。
vcl_recv
vcl_recv是在請求開始時調用的。完成該子程序后，請求就被接收并解析了。用于確定是否需要服務請求，怎么服務，如果可用，使用哪個后端。
在vcl_recv中，你也可以修改請求。通常你可以修改cookie，或添加/移除請求頭信息。
注意在vcl_recv中，只可以使用請求對象req。
vcl_fetch
vcl_fetch是在文檔從后端被成功接收后調用的。通常用于調整響應頭信息，觸發ESI處理，萬一請求失敗就換個后端服務器。
在vcl_fecth中，你還可以使用請求對象req。還有個后端響應對象beresp。Beresp包含了后端的HTTP頭信息。
varnish 3.X 配置參考文檔：http://anykoro.sinaapp.com/?p=261
編輯：/usr/local/varnish/vcl.conf ，文件不存在則創建。注意：Server A 與server B 配置一致！
配置詳情如下：
?
###########后臺代理服務器########
backend server_osapub_com{
.host = "192.168.4.97";
.port = "82";
}
acl purge {
"localhost";
"127.0.0.1";
}
#############################################
sub vcl_recv {
#################BAN##########################begin
if (req.request == "BAN") {
# Same ACL check as above:
if (!client.ip ~ purge) {
error 405 "Not allowed.";
}
ban("req.http.host == " + req.http.host +
"&& req.url == " + req.url);
error 200 "Ban added";
}
#################BAN##########################end
###############配置域名##################################
if(req.http.host ~ "^server.osapub.com" ||req.http.host ~ ".osapub.com" ) {
#使用哪一組后臺服務器
set req.backend = server_osapub_com;
}
##################################################
if (req.restarts == 0) {
if (req.http.x-forwarded-for) {
set req.http.X-Forwarded-For = req.http.X-Forwarded-For + ", " + client.ip;
}
else {
set req.http.X-Forwarded-For = client.ip;
}
}
if (req.http.Accept-Encoding) {
if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf)$") {
remove req.http.Accept-Encoding;
} elsif (req.http.Accept-Encoding ~ "gzip") {
set req.http.Accept-Encoding = "gzip";}
elsif (req.http.Accept-Encoding ~ "deflate") {
set req.http.Accept-Encoding = "deflate";}
else {
remove req.http.Accept-Encoding;
}
}
if (req.http.Cache-Control ~ "no-cache") {
return (pass);
}
if (req.request != "GET" &&
req.request != "HEAD" &&
req.request != "PUT" &&
req.request != "POST" &&
req.request != "TRACE" &&
req.request != "OPTIONS" &&
req.request != "DELETE") {
return (pipe);
}
if (req.request != "GET" && req.request != "HEAD") {
return(pass);
}
if (req.http.Authorization || req.http.Cookie ||req.http.Authenticate) {
return (pass);
}
if (req.request == "GET" && req.url ~ "(?i)\.php($|\?)"){
return (pass);
}
if (req.url ~ "\.(css|jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf)$") {
unset req.http.Cookie;
}
return (lookup);
}
sub vcl_fetch {
set beresp.grace = 5m;
if (beresp.status == 404 || beresp.status == 503 || beresp.status == 500 || beresp.status == 502) {
set beresp.http.X-Cacheable = "NO: beresp.status";
set beresp.http.X-Cacheable-status = beresp.status;
return (hit_for_pass);
}
#決定哪些頭不緩存
if (req.url ~ "\.(php|shtml|asp|aspx|jsp|js|ashx)$") {
return (hit_for_pass);
}
if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf)$") {
unset beresp.http.set-cookie;
}
if (beresp.ttl <= 0s) {
set beresp.http.X-Cacheable = "NO: !beresp.cacheable";
return (hit_for_pass);
}
else {
unset beresp.http.expires;
}
return (deliver);
}
sub vcl_deliver {
if (resp.http.magicmarker) {
unset resp.http.magicmarker;
set resp.http.age = "0";
}
# add cache hit data
if (obj.hits > 0) {
set resp.http.X-Cache = "HIT";
set resp.http.X-Cache-Hits = obj.hits;
}
else {
set resp.http.X-Cache = "MISS";
}
# hidden some sensitive http header returning to client, when the cache server received from backend server response
remove resp.http.X-Varnish;
remove resp.http.Via;
remove resp.http.Age;
remove resp.http.X-Powered-By;
remove resp.http.X-Drupal-Cache;
return (deliver);
}
sub vcl_error {
if (obj.status == 503 && req.restarts < 5) {
set obj.http.X-Restarts = req.restarts;
return (restart);
}
}
sub vcl_hit {
if (req.http.Cache-Control ~ "no-cache") {
if (! (req.http.Via || req.http.User-Agent ~ "bot|MSIE")) {
set obj.ttl = 0s;
return (restart);
}
}
return(deliver);
} 配置完成后執行：/root/varnish_start.sh 如果啟動成功則表示配置無誤！
四、 配置nginx編輯： /usr/local/nginx/conf/nginx.conf ，nginx 的配置方法請參考網上文檔資料。
Server A（192.168.4.97） nginx配置如下：
?
user www www;
worker_processes 16;
error_log /logs/nginx/nginx_error.log crit;
#Specifies the value for maximum file descriptors that can be opened by this process.
worker_rlimit_nofile 65500;
events
{
use epoll;
worker_connections 65500;
}
http
{
include mime.types;
default_type application/octet-stream;
server_names_hash_bucket_size 128;
client_header_buffer_size 64k;
large_client_header_buffers 4 64k;
client_max_body_size 10m;
server_tokens off;
expires 1h;
sendfile on;
tcp_nopush on;
keepalive_timeout 60;
tcp_nodelay on;
fastcgi_connect_timeout 200;
fastcgi_send_timeout 300;
fastcgi_read_timeout 600;
fastcgi_buffer_size 128k;
fastcgi_buffers 4 64k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 128k;
fastcgi_temp_path /dev/shm;
gzip on;
gzip_min_length 2048;
gzip_buffers 4 16k;
gzip_http_version 1.1;
gzip_types text/plain text/css application/xml application/x-javascript ;
log_format access '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" $http_x_forwarded_for';
################# include ###################
upstream varnish_server {
server 127.0.0.1:81 weight=2 max_fails=3 fail_timeout=30s;
server 192.168.4.99:81 weight=2 max_fails=3 fail_timeout=30s;
ip_hash;
}
server
{
listen 80;
server_name 192.168.4.96 192.168.4.98;
index index.html index.htm index.php index.aspx;
location /
{
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_connect_timeout 200;
proxy_send_timeout 300;
proxy_read_timeout 500;
proxy_buffer_size 256k;
proxy_buffers 4 128k;
proxy_busy_buffers_size 256k;
proxy_temp_file_write_size 256k;
proxy_temp_path /dev/shm;
proxy_pass http://varnish_server;
expires off;
access_log /logs/nginx/192.168.4.96.log access;
}
}
server
{
listen 82;
server_name 192.168.4.97;
index index.html index.htm index.php;
root /data/web/awstats/www;
location ~ .*\.php$
{
include fcgi.conf;
fastcgi_pass 127.0.0.1:10080;
fastcgi_index index.php;
expires off;
}
access_log /logs/nginx/awstats.osapub.com.log access;
}
} server B （192.168.4.99）配置如下：
?
user www www;
worker_processes 16;
error_log /logs/nginx/nginx_error.log crit;
#Specifies the value for maximum file descriptors that can be opened by this process.
worker_rlimit_nofile 65500;
events
{
use epoll;
worker_connections 65500;
}
http
{
include mime.types;
default_type application/octet-stream;
server_names_hash_bucket_size 128;
client_header_buffer_size 64k;
large_client_header_buffers 4 64k;
client_max_body_size 10m;
server_tokens off;
expires 1h;
sendfile on;
tcp_nopush on;
keepalive_timeout 60;
tcp_nodelay on;
fastcgi_connect_timeout 200;
fastcgi_send_timeout 300;
fastcgi_read_timeout 600;
fastcgi_buffer_size 128k;
fastcgi_buffers 4 64k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 128k;
fastcgi_temp_path /dev/shm;
gzip on;
gzip_min_length 2048;
gzip_buffers 4 16k;
gzip_http_version 1.1;
gzip_types text/plain text/css application/xml application/x-javascript ;
log_format access '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" $http_x_forwarded_for';
################# include ###################
upstream varnish_server {
server 127.0.0.1:81 weight=2 max_fails=3 fail_timeout=30s;
server 192.168.4.97:81 weight=2 max_fails=3 fail_timeout=30s;
ip_hash;
}
server
{
listen 80;
server_name 192.168.4.96 192.168.4.98;
index index.html index.htm index.php index.aspx;
location /
{
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_connect_timeout 200;
proxy_send_timeout 300;
proxy_read_timeout 500;
proxy_buffer_size 256k;
proxy_buffers 4 128k;
proxy_busy_buffers_size 256k;
proxy_temp_file_write_size 256k;
proxy_temp_path /dev/shm;
proxy_pass http://varnish_server;
expires off;
access_log /logs/nginx/192.168.4.98.log access;
}
}
server
{
listen 82;
server_name 192.168.4.99;
index index.html index.htm index.php;
root /data/web/awstats/www;
location ~ .*\.php$
{
include fcgi.conf;
fastcgi_pass 127.0.0.1:10080;
fastcgi_index index.php;
expires off;
}
access_log /logs/nginx/awstats.osapub.com.log access;
}
} 啟動nginx:
/root/nginx_start
啟動正常會監聽80，82端口！
五、 運行nginx ha腳本
server A 的腳本下載地址：http://bbs.osapub.com/down/server_a_ha.tar.gz
解壓力后得到三個腳本：
nginx_watchdog.sh
nginxha.sh
nginx_ha1.sh
server B 的腳本下載地址：http://bbs.osapub.com/down/server_b_ha.tar.gz
解壓力后得到三個腳本：
nginx_watchdog.sh
nginxha.sh
nginx_ha2.sh
注意：腳本建議放到：/data/sh/ha 目錄下，否則需要修改nginxha.sh 里面的程序路徑！
六、 測試1，測試之前先把nginx 負載去掉,可以在前面加#號暫時注釋，只保留本機解析，確保測試結果準確。
2，修改本機host文件
添加：192.168.4.99 server.osapub.com 到末尾，保存后訪問。
3，正常結果顯示如下：
到這里整個架構基本能運行起來了，根據大家的實際需求，對配置文件進行調優，HA腳本也可以進一步調優，關于報警，請參考社區自動安裝mutt報警的腳本！
本文由用戶 jopen 自行上傳分享，僅供網友學習交流。所有權歸原作者，若您的權利被侵害，請聯系管理員。
轉載本站原創文章，請注明出處，并保留原始鏈接、圖片水印。
本站是一個以用戶分享為主的開源技術平臺，歡迎各類分享！