A-A+

ELKStack 安装配置

2016年08月18日 运维自动化 暂无评论 阅读 1,379 次

包准备:

    操作系统环境 centos 6.7 final
    kibana-4.5.4
    logstatsh   2.3.4
    elasticsearch 2.3.5
    jdk1.7
一、配置ElasticSearch    
 解压 elasticsearch  并安装head
[root@ops1 elk]# ln -s elasticsearch-2.3.5 elasticsearch
[root@ops1 elk]# ll
总用量 26908
lrwxrwxrwx 1 root root       19 8月  11 13:02 elasticsearch -> elasticsearch-2.3.5
drwxr-xr-x 6 root root     4096 8月  11 13:02 elasticsearch-2.3.5
-rw-r--r-- 1 root root 27547995 8月  11 12:57 elasticsearch-2.3.5.tar.gz
[root@ops1 elk]# cd elasticsearch
[root@ops1 elasticsearch]# ll
总用量 44
drwxr-xr-x 2 root root  4096 8月  11 13:02 bin
drwxr-xr-x 2 root root  4096 8月  11 13:02 config
drwxr-xr-x 2 root root  4096 8月  11 13:02 lib
-rw-rw-r-- 1 1000 1000 11358 7月  27 18:34 LICENSE.txt
drwxrwxr-x 5 1000 1000  4096 7月  27 18:44 modules
-rw-rw-r-- 1 1000 1000   150 7月  27 18:34 NOTICE.txt
-rw-rw-r-- 1 1000 1000  8700 7月  27 18:34 README.textile
[root@ops1 elasticsearch]# ./bin/plugin install mobz/elasticsearch-head
-> Installing mobz/elasticsearch-head...
Plugins directory [/home/ops/elk/elasticsearch/plugins] does not exist. Creating...
Trying https://github.com/mobz/elasticsearch-head/archive/master.zip ...
Downloading ....................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................DONE
Verifying https://github.com/mobz/elasticsearch-head/archive/master.zip checksums if available ...
NOTE: Unable to verify checksum for downloaded plugin (unable to find .sha1 or .md5 file to verify)
Installed head into /home/ops/elk/elasticsearch/plugins/head
PS:增加用户
useradd elkstack
passwd elkstack 配置密码
然后配置sudo  如果没有sudo 
yum install sudo -y
然后执行vimsudo
 在root 下面增加一行
elkstack    ALL=(ALL)   ALL

下面是执行日志:
[root@ops1 bin]# su elkstack
[elkstack@ops1 bin]$ ll
总用量 324
-rwxr-xr-x 1 elkstack elkstack   5551 7月  27 18:34 elasticsearch
-rw-rw-r-- 1 elkstack elkstack    909 7月  27 18:34 elasticsearch.bat
-rw-rw-r-- 1 elkstack elkstack   3307 7月  27 18:34 elasticsearch.in.bat
-rwxr-xr-x 1 elkstack elkstack   2814 7月  27 18:34 elasticsearch.in.sh
-rw-rw-r-- 1 elkstack elkstack 104448 7月  27 00:09 elasticsearch-service-mgr.exe
-rw-rw-r-- 1 elkstack elkstack 103936 7月  27 00:09 elasticsearch-service-x64.exe
-rw-rw-r-- 1 elkstack elkstack  80896 7月  27 00:09 elasticsearch-service-x86.exe
-rwxr-xr-x 1 elkstack elkstack   2992 7月  27 18:34 plugin
-rw-rw-r-- 1 elkstack elkstack   1303 7月  27 18:34 plugin.bat
-rw-rw-r-- 1 elkstack elkstack   6501 7月  27 18:34 service.bat
[elkstack@ops1 bin]$ ll
总用量 324
-rwxr-xr-x 1 elkstack elkstack   5551 7月  27 18:34 elasticsearch
-rw-rw-r-- 1 elkstack elkstack    909 7月  27 18:34 elasticsearch.bat
-rw-rw-r-- 1 elkstack elkstack   3307 7月  27 18:34 elasticsearch.in.bat
-rwxr-xr-x 1 elkstack elkstack   2814 7月  27 18:34 elasticsearch.in.sh
-rw-rw-r-- 1 elkstack elkstack 104448 7月  27 00:09 elasticsearch-service-mgr.exe
-rw-rw-r-- 1 elkstack elkstack 103936 7月  27 00:09 elasticsearch-service-x64.exe
-rw-rw-r-- 1 elkstack elkstack  80896 7月  27 00:09 elasticsearch-service-x86.exe
-rwxr-xr-x 1 elkstack elkstack   2992 7月  27 18:34 plugin
-rw-rw-r-- 1 elkstack elkstack   1303 7月  27 18:34 plugin.bat
-rw-rw-r-- 1 elkstack elkstack   6501 7月  27 18:34 service.bat
[elkstack@ops1 bin]$ ./elasticsearch
log4j:ERROR setFile(null,true) call failed.
java.io.FileNotFoundException: /tmp/elasticsearch/logs/91_cluster.log (权限不够)
at java.io.FileOutputStream.open0(Native Method)
at java.io.FileOutputStream.open(FileOutputStream.java:270)
at java.io.FileOutputStream.<init>(FileOutputStream.java:213)
at java.io.FileOutputStream.<init>(FileOutputStream.java:133)
at org.apache.log4j.FileAppender.setFile(FileAppender.java:294)
at org.apache.log4j.FileAppender.activateOptions(FileAppender.java:165)
at org.apache.log4j.DailyRollingFileAppender.activateOptions(DailyRollingFileAppender.java:223)
at org.apache.log4j.config.PropertySetter.activate(PropertySetter.java:307)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:172)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:104)
at org.apache.log4j.PropertyConfigurator.parseAppender(PropertyConfigurator.java:842)
at org.apache.log4j.PropertyConfigurator.parseCategory(PropertyConfigurator.java:768)
at org.apache.log4j.PropertyConfigurator.configureRootCategory(PropertyConfigurator.java:648)
at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:514)
at org.apache.log4j.PropertyConfigurator.configure(PropertyConfigurator.java:440)
at org.elasticsearch.common.logging.log4j.LogConfigurator.configure(LogConfigurator.java:128)
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:243)
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:35)
log4j:ERROR Either File or DatePattern options are not set for appender .
log4j:ERROR setFile(null,true) call failed.
java.io.FileNotFoundException: /tmp/elasticsearch/logs/91_cluster_deprecation.log (权限不够)
at java.io.FileOutputStream.open0(Native Method)
at java.io.FileOutputStream.open(FileOutputStream.java:270)
at java.io.FileOutputStream.<init>(FileOutputStream.java:213)
at java.io.FileOutputStream.<init>(FileOutputStream.java:133)
at org.apache.log4j.FileAppender.setFile(FileAppender.java:294)
at org.apache.log4j.FileAppender.activateOptions(FileAppender.java:165)
at org.apache.log4j.DailyRollingFileAppender.activateOptions(DailyRollingFileAppender.java:223)
at org.apache.log4j.config.PropertySetter.activate(PropertySetter.java:307)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:172)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:104)
at org.apache.log4j.PropertyConfigurator.parseAppender(PropertyConfigurator.java:842)
at org.apache.log4j.PropertyConfigurator.parseCategory(PropertyConfigurator.java:768)
at org.apache.log4j.PropertyConfigurator.parseCatsAndRenderers(PropertyConfigurator.java:672)
at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:516)
at org.apache.log4j.PropertyConfigurator.configure(PropertyConfigurator.java:440)
at org.elasticsearch.common.logging.log4j.LogConfigurator.configure(LogConfigurator.java:128)
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:243)
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:35)
log4j:ERROR Either File or DatePattern options are not set for appender [deprecation_log_file].
log4j:ERROR setFile(null,true) call failed.
java.io.FileNotFoundException: /tmp/elasticsearch/logs/91_cluster_index_indexing_slowlog.log (权限不够)
at java.io.FileOutputStream.open0(Native Method)
at java.io.FileOutputStream.open(FileOutputStream.java:270)
at java.io.FileOutputStream.<init>(FileOutputStream.java:213)
at java.io.FileOutputStream.<init>(FileOutputStream.java:133)
at org.apache.log4j.FileAppender.setFile(FileAppender.java:294)
at org.apache.log4j.FileAppender.activateOptions(FileAppender.java:165)
at org.apache.log4j.DailyRollingFileAppender.activateOptions(DailyRollingFileAppender.java:223)
at org.apache.log4j.config.PropertySetter.activate(PropertySetter.java:307)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:172)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:104)
at org.apache.log4j.PropertyConfigurator.parseAppender(PropertyConfigurator.java:842)
at org.apache.log4j.PropertyConfigurator.parseCategory(PropertyConfigurator.java:768)
at org.apache.log4j.PropertyConfigurator.parseCatsAndRenderers(PropertyConfigurator.java:672)
at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:516)
at org.apache.log4j.PropertyConfigurator.configure(PropertyConfigurator.java:440)
at org.elasticsearch.common.logging.log4j.LogConfigurator.configure(LogConfigurator.java:128)
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:243)
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:35)
log4j:ERROR Either File or DatePattern options are not set for appender [index_indexing_slow_log_file].
log4j:ERROR setFile(null,true) call failed.
java.io.FileNotFoundException: /tmp/elasticsearch/logs/91_cluster_index_search_slowlog.log (权限不够)
at java.io.FileOutputStream.open0(Native Method)
at java.io.FileOutputStream.open(FileOutputStream.java:270)
at java.io.FileOutputStream.<init>(FileOutputStream.java:213)
at java.io.FileOutputStream.<init>(FileOutputStream.java:133)
at org.apache.log4j.FileAppender.setFile(FileAppender.java:294)
at org.apache.log4j.FileAppender.activateOptions(FileAppender.java:165)
at org.apache.log4j.DailyRollingFileAppender.activateOptions(DailyRollingFileAppender.java:223)
at org.apache.log4j.config.PropertySetter.activate(PropertySetter.java:307)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:172)
at org.apache.log4j.config.PropertySetter.setProperties(PropertySetter.java:104)
at org.apache.log4j.PropertyConfigurator.parseAppender(PropertyConfigurator.java:842)
at org.apache.log4j.PropertyConfigurator.parseCategory(PropertyConfigurator.java:768)
at org.apache.log4j.PropertyConfigurator.parseCatsAndRenderers(PropertyConfigurator.java:672)
at org.apache.log4j.PropertyConfigurator.doConfigure(PropertyConfigurator.java:516)
at org.apache.log4j.PropertyConfigurator.configure(PropertyConfigurator.java:440)
at org.elasticsearch.common.logging.log4j.LogConfigurator.configure(LogConfigurator.java:128)
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:243)
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:35)
log4j:ERROR Either File or DatePattern options are not set for appender [index_search_slow_log_file].
^C[elkstack@ops1 bin]$ sudo chown -R elkstack:elkstack /tmp/elasticsearch/
bash: sudo: command not found
[elkstack@ops1 bin]$ su
密码:
su: 密码不正确
[elkstack@ops1 bin]$ 91power@123
bash: 91power@123: command not found
[elkstack@ops1 bin]$ su
密码:
[root@ops1 bin]# chown -R elkstack:elkstack /tmp/elasticsearch/
[root@ops1 bin]# su elkstack
[elkstack@ops1 bin]$ ll
总用量 324
-rwxr-xr-x 1 elkstack elkstack   5551 7月  27 18:34 elasticsearch
-rw-rw-r-- 1 elkstack elkstack    909 7月  27 18:34 elasticsearch.bat
-rw-rw-r-- 1 elkstack elkstack   3307 7月  27 18:34 elasticsearch.in.bat
-rwxr-xr-x 1 elkstack elkstack   2814 7月  27 18:34 elasticsearch.in.sh
-rw-rw-r-- 1 elkstack elkstack 104448 7月  27 00:09 elasticsearch-service-mgr.exe
-rw-rw-r-- 1 elkstack elkstack 103936 7月  27 00:09 elasticsearch-service-x64.exe
-rw-rw-r-- 1 elkstack elkstack  80896 7月  27 00:09 elasticsearch-service-x86.exe
-rwxr-xr-x 1 elkstack elkstack   2992 7月  27 18:34 plugin
-rw-rw-r-- 1 elkstack elkstack   1303 7月  27 18:34 plugin.bat
-rw-rw-r-- 1 elkstack elkstack   6501 7月  27 18:34 service.bat
[elkstack@ops1 bin]$ ./elasticsearch
[2016-08-11 13:17:23,106][WARN ][bootstrap                ] unable to install syscall filter: seccomp unavailable: CONFIG_SECCOMP not compiled into kernel, CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER are needed
[2016-08-11 13:17:23,730][INFO ][node                     ] [node0] version[2.3.5], pid[3471], build[90f439f/2016-07-27T10:36:52Z]
[2016-08-11 13:17:23,730][INFO ][node                     ] [node0] initializing ...
[2016-08-11 13:17:25,539][INFO ][plugins                  ] [node0] modules [reindex, lang-expression, lang-groovy], plugins [head], sites [head]
[2016-08-11 13:17:25,602][INFO ][env                      ] [node0] using [1] data paths, mounts [[/ (/dev/mapper/vg_ops1-lv_root)]], net usable_space [41.6gb], net total_space [49gb], spins? [possibly], types [ext4]
[2016-08-11 13:17:25,602][INFO ][env                      ] [node0] heap size [990.7mb], compressed ordinary object pointers [true]
[2016-08-11 13:17:25,602][WARN ][env                      ] [node0] max file descriptors [4096] for elasticsearch process likely too low, consider increasing to at least [65536]
[2016-08-11 13:17:29,564][INFO ][node                     ] [node0] initialized
[2016-08-11 13:17:29,565][INFO ][node                     ] [node0] starting ...
Exception in thread "main" BindTransportException[Failed to resolve host null]; nested: UnknownHostException[ops1: unknown error];
Likely root cause: java.net.UnknownHostException: ops1: unknown error
at java.net.Inet6AddressImpl.lookupAllHostAddr(Native Method)
at java.net.InetAddress$2.lookupAllHostAddr(InetAddress.java:928)
at java.net.InetAddress.getAddressesFromNameService(InetAddress.java:1323)
at java.net.InetAddress.getAllByName0(InetAddress.java:1276)
at java.net.InetAddress.getAllByName(InetAddress.java:1192)
at java.net.InetAddress.getAllByName(InetAddress.java:1126)
at org.elasticsearch.common.network.NetworkService.resolveInternal(NetworkService.java:266)
at org.elasticsearch.common.network.NetworkService.resolveInetAddresses(NetworkService.java:209)
at org.elasticsearch.common.network.NetworkService.resolveBindHostAddresses(NetworkService.java:122)
at org.elasticsearch.transport.netty.NettyTransport.bindServerBootstrap(NettyTransport.java:424)
at org.elasticsearch.transport.netty.NettyTransport.doStart(NettyTransport.java:321)
at org.elasticsearch.common.component.AbstractLifecycleComponent.start(AbstractLifecycleComponent.java:68)
at org.elasticsearch.transport.TransportService.doStart(TransportService.java:182)
at org.elasticsearch.common.component.AbstractLifecycleComponent.start(AbstractLifecycleComponent.java:68)
at org.elasticsearch.node.Node.start(Node.java:278)
at org.elasticsearch.bootstrap.Bootstrap.start(Bootstrap.java:206)
at org.elasticsearch.bootstrap.Bootstrap.init(Bootstrap.java:272)
at org.elasticsearch.bootstrap.Elasticsearch.main(Elasticsearch.java:35)
Refer to the log for complete error details.
[2016-08-11 13:17:29,919][INFO ][node                     ] [node0] stopping ...
[2016-08-11 13:17:29,926][INFO ][node                     ] [node0] stopped
[2016-08-11 13:17:29,926][INFO ][node                     ] [node0] closing ...
[2016-08-11 13:17:29,949][INFO ][node                     ] [node0] closed
[elkstack@ops1 bin]$ vim /etc/hosts
[elkstack@ops1 bin]$ su
密码:
[root@ops1 bin]# vim /etc/ss
ssh/ ssl/ 
[root@ops1 bin]# vim /etc/ssh/
moduli                sshd_config           ssh_host_dsa_key.pub  ssh_host_key.pub      ssh_host_rsa_key.pub
ssh_config            ssh_host_dsa_key      ssh_host_key          ssh_host_rsa_key      
[root@ops1 bin]# vim /etc/ssh/ssh
ssh_config            ssh_host_dsa_key      ssh_host_key          ssh_host_rsa_key      
sshd_config           ssh_host_dsa_key.pub  ssh_host_key.pub      ssh_host_rsa_key.pub  
[root@ops1 bin]# vim /etc/ssh/ssh
ssh_config            ssh_host_dsa_key      ssh_host_key          ssh_host_rsa_key      
sshd_config           ssh_host_dsa_key.pub  ssh_host_key.pub      ssh_host_rsa_key.pub  
[root@ops1 bin]# vim /etc/ssh/sshd_config 
[root@ops1 bin]# visudo /etc/sudoers
bash: visudo: command not found
[root@ops1 bin]# visudo
bash: visudo: command not found
[root@ops1 bin]# vimsudo
bash: vimsudo: command not found
[root@ops1 bin]# visudoer
bash: visudoer: command not found
[root@ops1 bin]# vi
vigr      vim       vimdiff   vimtutor  vipw      
[root@ops1 bin]# vi
vigr      vim       vimdiff   vimtutor  vipw      
[root@ops1 bin]# s
Display all 124 possibilities? (y or n)
[root@ops1 bin]# su
su       suexec   sulogin  sum      sushell  suspend  
[root@ops1 bin]# su
su       suexec   sulogin  sum      sushell  suspend  
[root@ops1 bin]# visudo
bash: visudo: command not found
[root@ops1 bin]# yum install sudo -y
已加载插件:fastestmirror
设置安装进程
Loading mirror speeds from cached hostfile
epel/metalink                                                                             | 4.8 kB     00:00     
 * base: mirrors.neusoft.edu.cn
 * epel: mirrors.neusoft.edu.cn
 * extras: mirrors.tuna.tsinghua.edu.cn
 * updates: mirrors.tuna.tsinghua.edu.cn
base                                                                                      | 3.7 kB     00:00     
epel                                                                                      | 4.3 kB     00:00     
epel/primary_db                                                                           | 5.9 MB     00:10     
extras                                                                                    | 3.4 kB     00:00     
updates                                                                                   | 3.4 kB     00:00     
解决依赖关系
--> 执行事务检查
---> Package sudo.x86_64 0:1.8.6p3-24.el6 will be 安装
--> 处理依赖关系 vim-minimal,它被软件包 sudo-1.8.6p3-24.el6.x86_64 需要
--> 执行事务检查
---> Package vim-minimal.x86_64 2:7.4.629-5.el6 will be 安装
--> 完成依赖关系计算
依赖关系解决
=================================================================================================================
 软件包                      架构                   版本                              仓库                  大小
=================================================================================================================
正在安装:
 sudo                        x86_64                 1.8.6p3-24.el6                    base                 710 k
为依赖而安装:
 vim-minimal                 x86_64                 2:7.4.629-5.el6                   base                 422 k
事务概要
=================================================================================================================
Install       2 Package(s)
总下载量:1.1 M
Installed size: 3.2 M
下载软件包:
(1/2): sudo-1.8.6p3-24.el6.x86_64.rpm                                                     | 710 kB     00:01     
(2/2): vim-minimal-7.4.629-5.el6.x86_64.rpm                                               | 422 kB     00:00     
-----------------------------------------------------------------------------------------------------------------
总计                                                                             488 kB/s | 1.1 MB     00:02     
运行 rpm_check_debug 
执行事务测试
事务测试成功
执行事务
  正在安装   : 2:vim-minimal-7.4.629-5.el6.x86_64                                                            1/2 
  正在安装   : sudo-1.8.6p3-24.el6.x86_64                                                                    2/2 
  Verifying  : sudo-1.8.6p3-24.el6.x86_64                                                                    1/2 
  Verifying  : 2:vim-minimal-7.4.629-5.el6.x86_64                                                            2/2 
已安装:
  sudo.x86_64 0:1.8.6p3-24.el6                                                                                   
作为依赖被安装:
  vim-minimal.x86_64 2:7.4.629-5.el6                                                                             
完毕!
[root@ops1 bin]# vim /etc/sysconfig/iptables
[root@ops1 bin]# visudu
bash: visudu: command not found
[root@ops1 bin]# whereis sudo
sudo: /usr/bin/sudo /etc/sudo.conf /usr/share/man/man8/sudo.8.gz
[root@ops1 bin]# sudo
usage: sudo [-D level] -h | -K | -k | -V
usage: sudo -v [-AknS] [-D level] [-g groupname|#gid] [-p prompt] [-u user name|#uid]
usage: sudo -l[l] [-AknS] [-D level] [-g groupname|#gid] [-p prompt] [-U user name] [-u user name|#uid] [-g
            groupname|#gid] [command]
usage: sudo [-AbEHknPS] [-r role] [-t type] [-C fd] [-D level] [-g groupname|#gid] [-p prompt] [-u user
            name|#uid] [-g groupname|#gid] [VAR=value] [-i|-s] [<command>]
usage: sudo -e [-AknS] [-r role] [-t type] [-C fd] [-D level] [-g groupname|#gid] [-p prompt] [-u user
            name|#uid] file ...
[root@ops1 bin]# v
vdir           vgconvert      vgimport       vgremove       vi             vimtutor       
vgcfgbackup    vgcreate       vgimportclone  vgrename       view           vipw           
vgcfgrestore   vgdisplay      vgmerge        vgs            vigr           visudo         
vgchange       vgexport       vgmknodes      vgscan         vim            vmstat         
vgck           vgextend       vgreduce       vgsplit        vimdiff        
[root@ops1 bin]# v
vdir           vgconvert      vgimport       vgremove       vi             vimtutor       
vgcfgbackup    vgcreate       vgimportclone  vgrename       view           vipw           
vgcfgrestore   vgdisplay      vgmerge        vgs            vigr           visudo         
vgchange       vgexport       vgmknodes      vgscan         vim            vmstat         
vgck           vgextend       vgreduce       vgsplit        vimdiff        
[root@ops1 bin]# vi
vi        view      vigr      vim       vimdiff   vimtutor  vipw      visudo    
[root@ops1 bin]# vi
vi        view      vigr      vim       vimdiff   vimtutor  vipw      visudo    
[root@ops1 bin]# visudo
[root@ops1 bin]# chgrp elkstach -g wheel
chgrp:无效选项 -- g
请尝试执行"chgrp --help"来获取更多信息。
[root@ops1 bin]# chgrp elkstach -G wheel
chgrp:无效选项 -- G
请尝试执行"chgrp --help"来获取更多信息。
[root@ops1 bin]# chgrp elkstach wheel
chgrp: 无效的组:"elkstach"
[root@ops1 bin]# chgrp elkstack wheel
chgrp: 无法访问"wheel": 没有那个文件或目录
[root@ops1 bin]# chgrp elkstack ^Ceel
[root@ops1 bin]# visudo
[root@ops1 bin]# su elkstack
[elkstack@ops1 bin]$ sudo ifconfig
We trust you have received the usual lecture from the local System
Administrator. It usually boils down to these three things:
    #1) Respect the privacy of others.
    #2) Think before you type.
    #3) With great power comes great responsibility.
[sudo] password for elkstack: 
eth0      Link encap:Ethernet  HWaddr 00:0C:29:F6:21:CB  
          inet addr:192.168.0.91  Bcast:192.168.0.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fef6:21cb/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:597771131 errors:0 dropped:0 overruns:0 frame:0
          TX packets:1886595 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:49296973078 (45.9 GiB)  TX bytes:2328447066 (2.1 GiB)
lo        Link encap:Local Loopback  
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:93 errors:0 dropped:0 overruns:0 frame:0
          TX packets:93 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:10714 (10.4 KiB)  TX bytes:10714 (10.4 KiB)
[elkstack@ops1 bin]$ ifconfig
eth0      Link encap:Ethernet  HWaddr 00:0C:29:F6:21:CB  
          inet addr:192.168.0.91  Bcast:192.168.0.255  Mask:255.255.255.0
          inet6 addr: fe80::20c:29ff:fef6:21cb/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:597786061 errors:0 dropped:0 overruns:0 frame:0
          TX packets:1886606 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:49298167828 (45.9 GiB)  TX bytes:2328449356 (2.1 GiB)
lo        Link encap:Local Loopback  
          inet addr:127.0.0.1  Mask:255.0.0.0
          inet6 addr: ::1/128 Scope:Host
          UP LOOPBACK RUNNING  MTU:65536  Metric:1
          RX packets:93 errors:0 dropped:0 overruns:0 frame:0
          TX packets:93 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0 
          RX bytes:10714 (10.4 KiB)  TX bytes:10714 (10.4 KiB)
[elkstack@ops1 bin]$ netstat -tnlp
(No info could be read for "-p": geteuid()=500 but you should be root.)
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address               Foreign Address             State       PID/Program name   
tcp        0      0 0.0.0.0:3690                0.0.0.0:*                   LISTEN      -                   
tcp        0      0 127.0.0.1:25                0.0.0.0:*                   LISTEN      -                   
tcp        0      0 0.0.0.0:3322                0.0.0.0:*                   LISTEN      -                   
tcp        0      0 ::ffff:127.0.0.1:8005       :::*                        LISTEN      -                   
tcp        0      0 :::8009                     :::*                        LISTEN      -                   
tcp        0      0 :::8080                     :::*                        LISTEN      -                   
tcp        0      0 :::80                       :::*                        LISTEN      -                   
tcp        0      0 ::1:25                      :::*                        LISTEN      -                   
tcp        0      0 :::3322                     :::*                        LISTEN      -                   
[elkstack@ops1 bin]$ vim /etc/hosts
[elkstack@ops1 bin]$ sudo vim /etc/hosts
[elkstack@ops1 bin]$ ll
总用量 324
-rwxr-xr-x 1 elkstack elkstack   5551 7月  27 18:34 elasticsearch
-rw-rw-r-- 1 elkstack elkstack    909 7月  27 18:34 elasticsearch.bat
-rw-rw-r-- 1 elkstack elkstack   3307 7月  27 18:34 elasticsearch.in.bat
-rwxr-xr-x 1 elkstack elkstack   2814 7月  27 18:34 elasticsearch.in.sh
-rw-rw-r-- 1 elkstack elkstack 104448 7月  27 00:09 elasticsearch-service-mgr.exe
-rw-rw-r-- 1 elkstack elkstack 103936 7月  27 00:09 elasticsearch-service-x64.exe
-rw-rw-r-- 1 elkstack elkstack  80896 7月  27 00:09 elasticsearch-service-x86.exe
-rwxr-xr-x 1 elkstack elkstack   2992 7月  27 18:34 plugin
-rw-rw-r-- 1 elkstack elkstack   1303 7月  27 18:34 plugin.bat
-rw-rw-r-- 1 elkstack elkstack   6501 7月  27 18:34 service.bat
[elkstack@ops1 bin]$ ./elasticsearch
[2016-08-11 13:26:44,640][WARN ][bootstrap                ] unable to install syscall filter: seccomp unavailable: CONFIG_SECCOMP not compiled into kernel, CONFIG_SECCOMP and CONFIG_SECCOMP_FILTER are needed
[2016-08-11 13:26:45,093][INFO ][node                     ] [node0] version[2.3.5], pid[3580], build[90f439f/2016-07-27T10:36:52Z]
[2016-08-11 13:26:45,094][INFO ][node                     ] [node0] initializing ...
[2016-08-11 13:26:46,176][INFO ][plugins                  ] [node0] modules [reindex, lang-expression, lang-groovy], plugins [head], sites [head]
[2016-08-11 13:26:46,224][INFO ][env                      ] [node0] using [1] data paths, mounts [[/ (/dev/mapper/vg_ops1-lv_root)]], net usable_space [41.6gb], net total_space [49gb], spins? [possibly], types [ext4]
[2016-08-11 13:26:46,224][INFO ][env                      ] [node0] heap size [990.7mb], compressed ordinary object pointers [true]
[2016-08-11 13:26:46,224][WARN ][env                      ] [node0] max file descriptors [4096] for elasticsearch process likely too low, consider increasing to at least [65536]
[2016-08-11 13:26:49,175][INFO ][node                     ] [node0] initialized
[2016-08-11 13:26:49,176][INFO ][node                     ] [node0] starting ...
[2016-08-11 13:26:49,378][INFO ][transport                ] [node0] publish_address {ops1/192.168.0.91:9300}, bound_addresses {192.168.0.91:9300}
[2016-08-11 13:26:49,393][INFO ][discovery                ] [node0] 91_cluster/crn6NyS9SI61CXYSnjjtxw
[2016-08-11 13:26:52,492][INFO ][cluster.service          ] [node0] new_master {node0}{crn6NyS9SI61CXYSnjjtxw}{192.168.0.91}{ops1/192.168.0.91:9300}, reason: zen-disco-join(elected_as_master, [0] joins received)
[2016-08-11 13:26:52,550][INFO ][http                     ] [node0] publish_address {ops1/192.168.0.91:9200}, bound_addresses {192.168.0.91:9200}
[2016-08-11 13:26:52,552][INFO ][node                     ] [node0] started
[2016-08-11 13:26:52,588][INFO ][gateway                  ] [node0] recovered [0] indices into cluster_state
Ctrl C 结束  
后台运行 你可以用screen  也可以 用./elasticsearch &   执行 回车即可。
打开浏览器
出现这个即可。 可以看到cluster_name 和name 以及安装的es的版本号
刚刚安装的head插件 它是一个用浏览器跟ES集群交互的插件,可以看集群状态、doc内容,执行搜索和普通的rest请求。可以输入http://ip:9200/_plugin/head 查看
这是我的页面
可以看到,现在Es集群种没有index 也没有type,因此这两条都是空的
es的配置容易出问题的就是用户方面,不能用root去执行 需要你创建一个普通用户,另外jdk版本,保证1.7以上。其他安装步骤走下来基本不会出问题。
二、下一部分 logstash安装配置
logstash功能如下;
数据收集,我们需要为它指定Input 和Output  (这两个可以为多个)。由于我么需要把java代码中log4j的日志输出到es(ElasticSearch,下面简称es)种,因此这里Inpu就是Log4j,而output自然就是es了。es做数据存储和搜索;

配置Logstash:

tar -zxvf logstash-2.1.1.tar.gzcd logstash-2.1.1

编写配置文件(名字和位置可以随意,这里我放在config目录下,取名为log4j_to_es.conf):

mkdir config
vi config/log4j_to_es.conf

输入以下内容:

# For detail structure of this file# Set: https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html
input {
  # For detail config for log4j as input, # See: https://www.elastic.co/guide/en/logstash/current/plugins-inputs-log4j.html
  log4j {
    mode => "server"
    host => "centos2"
    port => 4567
  }
}
filter {
  #Only matched data are send to output.
}
output {
  # For detail config for elasticsearch as output, # See: https://www.elastic.co/guide/en/logstash/current/plugins-outputs-elasticsearch.html
  elasticsearch {
    action => "index"          #The operation on ES
    hosts  => "centos2:9200"   #ElasticSearch host, can be array.
    index  => "applog"         #The index to write data to.
  }
}

logstash命令只有2个参数:

因此使用agent来启动它(使用-f指定配置文件):

./bin/logstash agent -f config/log4j_to_es.conf

到这里,我们已经可以使用Logstash来收集日志并保存到ES中了,下面来看看项目代码。

 

Java项目

照例先看项目结构图:

pom.xml,很简单,只用到了Log4j库:

<dependency><groupId>log4j</groupId><artifactId>log4j</artifactId><version>1.2.17</version></dependency>

 

log4j.properties,将Log4j的日志输出到SocketAppender,因为官网是这么说的:

log4j.rootLogger=INFO,console

# for package com.demo.elk, log would be sent to socket appender.log4j.logger.com.demo.elk=DEBUG, socket

# appender socketlog4j.appender.socket=org.apache.log4j.net.SocketAppender
log4j.appender.socket.Port=4567
log4j.appender.socket.RemoteHost=centos2
log4j.appender.socket.layout=org.apache.log4j.PatternLayout
log4j.appender.socket.layout.ConversionPattern=%d [%-5p] [%l] %m%n
log4j.appender.socket.ReconnectionDelay=10000

# appender consolelog4j.appender.console=org.apache.log4j.ConsoleAppender
log4j.appender.console.target=System.out
log4j.appender.console.layout=org.apache.log4j.PatternLayout
log4j.appender.console.layout.ConversionPattern=%d [%-5p] [%l] %m%n

注意:这里的端口号需要跟Logstash监听的端口号一致,这里是4567。

 

Application.java,使用Log4j的LOGGER打印日志即可:

package com.demo.elk;

import org.apache.log4j.Logger;

public class Application {
    private static final Logger LOGGER = Logger.getLogger(Application.class);
    public static void main(String[] args) throws Exception {
        for (int i = 0; i < 10; i++) {
            LOGGER.error("Info log [" + i + "].");
            Thread.sleep(500);
        }
    }
}

 

用Head插件查看ES状态和内容

运行Application.java,先看看console的输出(当然,这个输出只是为了做验证,不输出到console也可以的):

再来看看ES的head页面:

切换到Browser标签:

单击某一个文档(doc),则会展示该文档的所有信息:

可以看到,除了基础的message字段是我们的日志内容,Logstash还为我们增加了许多字段。而在https://www.elastic.co/guide/en/logstash/current/plugins-inputs-log4j.html中也明确说明了这一点:

上面使用了ES的Head插件观察了ES集群的状态和数据,但这只是个简单的用于跟ES交互的页面而已,并不能生成报表或者图表什么的,接下来使用Kibana来执行搜索并生成图表。

安装logstash
1、下载logstash
   logstatsh   2.3.4
 
2、安装logstash
下载下来的是tar.gz文件,直接tar zxf    logstatsh   2.3.4.tar.gz 即可

然后cd    logstatsh_2.3.4 && mkdir config

3、配置(默认没有这个配置文件)
1)配置logstash_agent
vim logstash_agent.conf
input {
        file {
                type => "messages"
                path => ["/var/log/messages"]
        }
        file {
                type => "elasticsearch"
                path => ['/var/log/elasticsearch/elasticsearch.log*']
        }
}
output {
        redis {
                host => "10.1.10.185"
                data_type => "list"
                key => "logstash:redis"
        }
}

ps:

上面这里 output可以不输出到redis 而是elasticSearch  配置就改成



output{

    elasticsearch {

        index => "syslog"    

        hosts => "192.168.0.91:9200"

        

    }



}
2)配置logstash_indexer(如果你用redis了 下面需要配置 没有用redis 下面不需要配置)

cat /etc/logstash/confiig logstash_indexer.conf
input {
        redis {
                host => "10.1.10.185"
                data_type => "list"
                key => "logstash:redis"
                type => "redis-input"
        port => "6379"
        }
}
output {
        elasticsearch {
                host => "10.1.10.185"
        }
}
 
4、启动服务
/home/ops/elk/logstatsh_2.3.4/bin agent -f /home/ops/elk/logstatsh_2.3.4/config/logstash_agent.conf

即可
 
5、使用jps -mlv或ps -ef来查看下进程
ps -ef|grep logst
logstash  22932      1 16 15:19 pts/0    00:00:01 /usr/bin/java -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -Djava.io.tmpdir=/var/lib/logstash -Xmx500m -Xss2048k -Djffi.boot.library.path=/opt/logstash/vendor/jruby/lib/jni -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -Djava.awt.headless=true -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -Djava.io.tmpdir=/var/lib/logstash -Xbootclasspath/a:/opt/logstash/vendor/jruby/lib/jruby.jar -classpath : -Djruby.home=/opt/logstash/vendor/jruby -Djruby.lib=/opt/logstash/vendor/jruby/lib -Djruby.script=jruby -Djruby.shell=/bin/sh org.jruby.Main --1.9 /opt/logstash/lib/bootstrap/environment.rb logstash/runner.rb agent -f /etc/logstash/conf.d -l /var/log/logstash/logstash.log

 
 安装kibana(前端web)
1、下载
 kibana-4.5.4.tar.gz
 
2、解压到指定目录
tar zxvf kibana-4.5.4-linux-x64.tar.gz /home/ops/elk/
 
3.编辑配置文件

cd kibana/config

vim kibana.yml

修改elasticsearch_url: "http:/192.168.0.91:9200"

然后 kibana/bin/kibana 启动即可。
查看端口

netstat -tupnl|grep 5601
tcp        0      0 10.1.10.185:5601        0.0.0.0:*              LISTEN      22982/node  
 
7、在windows上访问http://ip:5601

ELKstack日志分析平台

标签:
蜗牛的梦想
Copyright © Linux系统运维联盟 保留所有权利.   Theme  Ality 京公网安备11011402000199号 京ICP备14047887号

用户登录