日志分析系统ELK（业务日志）

所有日志文件目录在/data/visitlog下，按日期分子目录

1，上传所有所需包至服务器

elasticsearch-1.7.0.zip  

jdk-7u79-linux-x64.rpm  

kibana-4.1.1-linux-x64.tar.gz  

logstash-1.5.3.tar.gz

nginx-1.8.0.tar.gz

2,安装jdk

rpm -ivh nginx-1.8.0.tar.gz

echo export JAVA_HOME=/usr/java/jdk1.7.0_79/ >> /etc/profile

echo export PATH=$JAVA_HOME/bin:$PATH >> /etc/profile

echo export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar >> /etc/profile

source /etc/profile

3，部署elk

tar xzf kibana-4.1.1-linux-x64.tar.gz -C /data/elk/

tar xzf logstash-1.5.3.tar.gz -C /data/elk/

unzip elasticsearch-1.7.0.zip

mv elasticsearch-1.7.0 /data/elk/elasticsearch

cd /data/elk

mv kibana-4.1.1-linux-x64 kibana

mv logstash-1.5.3 logstash

4,配置logstash

mkdir /data/elk/logstash/etc

vim /data/elk/logstash/etc/logs.conf

input {
  file {
     path => ["/data/visitlog/**/*.log" ]
     #start_position => "beginning"   #start_position为从何处导入日志，不配置的情况下默认为从开启服务时开始导入生成的日志，beginning为将目录中所有日志导入
   }
}
output {
  stdout { codec=> dots }
  elasticsearch {host => "localhost" }
}

5,启动服务

nohup /data/elk/kibana/bin/kibana &

nohup /data/elk/elasticsearch/bin/elasticsearch &

nohup /data/elk/logstash/bin/logstash -f /data/elk/logstash/etc/log.conf &

6，查看是否有9200 9300 5601端口启动

7，登录kibana查看

http://IP:5601

8，nginx反向代理与认证登录配置

yum install pcre-devel zlib-devel -y

tar xzf nginx-1.8.0.tar.gz

cd nginx-1.8.0

./configure --prefix=/usr/local/nginx

make && make install

vim /etc/init.d/nginx

#!/bin/bash
# nginx Startup script for the Nginx HTTP Server
# this script create it by ruijie. at 2014.02.26
# if you find any errors on this scripts,please contact ruijie.
# and send mail to ruijie at gmail dot com.
#            [email protected]
### BEGIN INIT INFO
# Provides:          nginx
# Default-Start:     2 3 4 5
# Default-Stop:      0 1 6
# Short-Description: starts nginx
# Description:       starts the nginx server
### END INIT INFO


nginxd=/usr/local/nginx/sbin/nginx
nginx_config=/usr/local/nginx/conf/nginx.conf
nginx_pid=/usr/local/nginx/logs/nginx.pid


RETVAL=0
prog="nginx"


[ -x $nginxd ] || exit 0


# Start nginx daemons functions.
start() {


    if [ -e $nginx_pid ] && netstat -tunpl | grep nginx &> /dev/null;then
        echo "nginx already running...."
        exit 1
    fi

    echo -n $"Starting $prog!"
    $nginxd -c ${nginx_config}
    RETVAL=$?
    echo
    [ $RETVAL = 0 ] && touch /var/lock/nginx
    return $RETVAL
}

# Stop nginx daemons functions.
stop() {
    echo -n $"Stopping $prog!"
    $nginxd -s stop
    RETVAL=$?
    echo
    [ $RETVAL = 0 ] && rm -f /var/lock/nginx
}


# reload nginx service functions.
reload() {
    echo -n $"Reloading $prog!"
    #kill -HUP `cat ${nginx_pid}`
    $nginxd -s reload
    RETVAL=$?
    echo

}

# See how we were called.
case "$1" in
start)
        start
        ;;


stop)
        stop
        ;;


reload)
        reload
        ;;


restart)
        stop
        start
        ;;


*)
        echo $"Usage: $prog {start|stop|restart|reload|help}"
        exit 1
esac


exit $RETVAL

chmod +x /etc/init.d/nginx

mkdir /usr/local/nginx/conf/conf.d

nginx.conf http模块中添加include     /usr/local/nginx/conf/conf.d/*.conf; 注释掉server模块

vim /usr/local/nginx/conf/conf.d/kibana.conf

server {
        listen       80;
        server_name  localhost;
        #charset koi8-r;
        #access_log  logs/kibana.access.log  main;
        error_log   logs/kibana.error.log;
        location / {
            #root   html;
            #index  index.html index.htm;
            auth_basic "secret";
            auth_basic_user_file /usr/local/nginx/passwd.db;
            proxy_pass http://127.0.0.1:5601/;
            proxy_set_header   Cookie $http_cookie;
            #proxy_cookie_path  /vga/ /;
            proxy_set_header  X-Real-IP  $remote_addr;
            proxy_set_header  X-Forwarded-For  $proxy_add_x_forwarded_for;
            proxy_set_header  Host  $http_host;
        }
}

htpasswd -c /usr/local/nginx/passwd.db admin    #admin为登录用户

chmod 777 passwd.db

service nginx start

9，现在可直接使用IP地址来登录kibana，提示输入用户名密码