docker安装elk

阅读数: 次 2021-11-11

docker安装elk

双十一买了个新的性能高一点的服务器，重新装下elk，记录一下操作过程=.=

# 下载es并配置内存
[root@un1queyan ~]# docker run -d --name elasticsearch -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e ES_JAVA_OPTS="-Xms512m -Xmx1g" elasticsearch:7.6.2

# 测试下是否启动成功
[root@un1queyan ~]# curl localhost:9200
{
  "name" : "777a2871c206",
  "cluster_name" : "docker-cluster",
  "cluster_uuid" : "y-WOqLW5SUKk2S3FnYvgxA",
  "version" : {
    "number" : "7.6.2",
    "build_flavor" : "default",
    "build_type" : "docker",
    "build_hash" : "ef48eb35cf30adf4db14086e8aabd07ef6fb113f",
    "build_date" : "2020-03-26T06:34:37.794943Z",
    "build_snapshot" : false,
    "lucene_version" : "8.4.0",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}

head插件的安装

# 下载head镜像
[root@un1queyan ~]# docker pull mobz/elasticsearch-head:5

# 启动
[root@un1queyan ~]# docker run -d -p 9100:9100 docker.io/mobz/elasticsearch-head:5

启动了下发现跨域了，不使用谷歌跨域插件的话要修改下配置文件

1 2	# 查看下容器id 进去修改 [root@un1queyan ~]# docker ps

[root@un1queyan ~]# docker exec -it 777a2871c206 bash
[root@777a2871c206 elasticsearch]# cd ./config/
[root@777a2871c206 config]# ls
elasticsearch.keystore  elasticsearch.yml  jvm.options  log4j2.properties  role_mapping.yml  roles.yml  users  users_roles
[root@777a2871c206 config]# cat elasticsearch.yml 
cluster.name: "docker-cluster"
network.host: 0.0.0.0

# 执行shell命令直接追加 ~
[root@777a2871c206 config]# echo -e 'http.cors.enabled: true\nhttp.cors.allow-origin: "*" '>>elasticsearch.yml

# 添加成功
[root@777a2871c206 config]# cat elasticsearch.yml 
cluster.name: "docker-cluster"
network.host: 0.0.0.0
http.cors.enabled: true
http.cors.allow-origin: "*" 

# 退出去重启
[root@un1queyan ~]# docker restart 777a2871c206
777a2871c206

搞定！

安装kibana7.6.2及ik分词器

# 下载
[root@un1queyan ~]# docker pull kibana:7.6.2

# 运行
[root@un1queyan ~]# docker run --name kibana --link elasticsearch:elasticsearch -p 5601:5601 -d kibana:7.6.2
# 参数说明:
-p 5601:5601 将容器的5601端口映射到主机的5601端口;
--name kibana 给当前启动的容器取名叫 kibana
-d 以后台方式运行(daemon)
--link 可以用来链接2个容器，使得源容器（被链接的容器）和接收容器（主动去链接的容器）之间可以互相通信，并且接收容器可以获取源容器的一些数据，如源容器的环境变量。

# 获取es内部的ip
[root@un1queyan ~]# docker inspect --format '{{ .NetworkSettings.IPAddress }}' 777a2871c206
172.17.0.5


# 进入kibana容器内部修改配置
[root@un1queyan ~]# docker exec -it 08679925b731 /bin/bash
bash-4.2$ cd config/
bash-4.2$ ls
apm.js	kibana.yml
bash-4.2$ cat kibana.yml 
#
# ** THIS IS AN AUTO-GENERATED FILE **
#

# Default Kibana configuration for docker target
server.name: kibana
server.host: "0"
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
xpack.monitoring.ui.container.elasticsearch.enabled: truebash-4.2$

# shell小改下配置
echo -e 'server.name: kibana\nserver.host: "0"\nelasticsearch.hosts: [ "http://172.17.0.5:9200" ]\nxpack.monitoring.ui.container.elasticsearch.enabled: true\ni18n.locale: "zh-CN"'>kibana.yml

# 修改完成后查看一下
bash-4.2$ cat kibana.yml 
server.name: kibana
server.host: "0"
elasticsearch.hosts: [ "http://172.17.0.5:9200" ]
xpack.monitoring.ui.container.elasticsearch.enabled: true
i18n.locale: "zh-CN"

# 退出
bash-4.2$ exit
exit

# 重启
[root@un1queyan ~]# docker restart 08679925b731
08679925b731

测试kibana连接成功

安装ik分词插件

# ik分词器地址
https://github.com/medcl/elasticsearch-analysis-ik/

# 进入容器
[root@un1queyan ~]# docker exec -it 08679925b731 /bin/bash

#  在线安装
elasticsearch-plugin install https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.6.2/elasticsearch-analysis-ik-7.6.2.zip

[root@777a2871c206 elasticsearch]# cd plugins/
[root@777a2871c206 plugins]# ls
analysis-ik

测试成功！

docker安装logstash，并导入mysql数据到es

[root@un1queyan /]# docker pull logstash:7.6.2
[root@un1queyan /]# docker run -d --name=logstash logstash:7.6.2

[root@un1queyan /]# mkdir -p /data/docker/logstash
[root@un1queyan /]# mkdir -p /data/docker/logstash/mysql # 用于存放jdbc jar包
[root@un1queyan /]# mkdir -p /data/docker/logstash/template # 存放mysql to es 动态模板
[root@un1queyan /]# docker cp 5f70d31814c4:/usr/share/logstash/config /data/docker/logstash # 复制配置文件，冒号前面为容器ID

[root@un1queyan /]# docker cp 5f70d31814c4:/usr/share/logstash/pipeline /data/docker/logstash # 复制logstash管道文件

# 类似es启动限制内存大小
[root@un1queyan /]# docker run -di --name=logstash -e LS_JAVA_OPTS="-Xms512m -Xmx512m" -v /data/docker/logstash/config:/usr/share/logstash/config -v /data/docker/logstash/mysql:/usr/share/logstash/mysql -v /data/docker/logstash/template:/usr/share/logstash/template -v /data/docker/logstash/pipeline:/usr/share/logstash/pipeline logstash:7.6.2



#logstash挂载目录结构
[root@un1queyan logstash]# ls -R
.:
config  mysql  pipeline  template

./config:
jvm.options  log4j2.properties  logstash-sample.conf  logstash.yml  pipelines.yml  startup.options

./mysql:

./pipeline:
logstash.conf

./template:
# 下载mysql JDBC驱动jar包
https://dev.mysql.com/downloads/connector/j/?os=26 #下载的驱动放入mysql文件夹
 
# 配置logstash.conf,放入pipline文件夹
input {
        stdin {}
        jdbc {
                type => "articles"
                 # 数据库连接地址
                jdbc_connection_string => "jdbc:mysql://127.0.0.1:3306/DbBase?characterEncoding=UTF-8&autoReconnect=true"
                 # 数据库连接账号密码；
                jdbc_user => "username"
                jdbc_password => "pwd"
                 # MySQL依赖包路径；
                jdbc_driver_library => "/usr/share/logstash/mysql/mysql-connector-java-8.0.23.jar"
                 # the name of the driver class for mysql
                jdbc_driver_class => "com.mysql.jdbc.Driver"
                 # 数据库重连尝试次数
                connection_retry_attempts => "3"
                 # 判断数据库连接是否可用，默认false不开启
                jdbc_validate_connection => "true"
                 # 数据库连接可用校验超时时间，默认3600S
                jdbc_validation_timeout => "3600"
                 # 开启分页查询（默认false不开启）；
                jdbc_paging_enabled => "true"
                 # 单次分页查询条数（默认100000,若字段较多且更新频率较高，建议调低此值）；
                jdbc_page_size => "30"
                 # statement为查询数据sql，如果sql较复杂，建议配通过statement_filepath配置sql文件的存放路径；
                 # sql_last_value为内置的变量，存放上次查询结果中最后一条数据tracking_column的值，此处即为update_time；
                 # statement_filepath => "mysql/jdbc.sql"
                statement => "SELECT id,title,create_time,content,update_time,status FROM `scmy_articles` WHERE status=0 and update_time>= :sql_last_value order by create_time desc"
                 # 是否将字段名转换为小写，默认true（如果有数据序列化、反序列化需求，建议改为false）；
                lowercase_column_names => false
                 # Value can be any of: fatal,error,warn,info,debug，默认info；
                sql_log_level => warn
                 #
                 # 是否记录上次执行结果，true表示会将上次执行结果的tracking_column字段的值保存到last_run_metadata_path指定的文件中；
                record_last_run => true
                 # 需要记录查询结果某字段的值时，此字段为true，否则默认tracking_column为timestamp的值；
                use_column_value => true
                 # 需要记录的字段，用于增量同步，需是数据库字段
                tracking_column => "update_time"
                 # Value can be any of: numeric,timestamp，Default value is "numeric"
                tracking_column_type => timestamp
                 # record_last_run上次数据存放位置；
                last_run_metadata_path => "/usr/share/logstash/mysql/art_id.txt"
                 # 是否清除last_run_metadata_path的记录，需要增量同步时此字段必须为false；
                clean_run => false
                 #
                 # 设置监听间隔 各字段含义（从左至右）分、时、天、月、年，全为*默认含义为每分钟都更新
                schedule => "*/5 * * * *" #每5分钟更新一次
        }
   jdbc {
                type => "doc"
                 # 数据库连接地址
               jdbc_connection_string => "jdbc:mysql://127.0.0.1:3306/DbBase?characterEncoding=UTF-8&autoReconnect=true"
                 # 数据库连接账号密码；
                jdbc_user => "username"
                jdbc_password => "pwd"
                 # MySQL依赖包路径；
                jdbc_driver_library => "/usr/share/logstash/mysql/mysql-connector-java-8.0.23.jar"
                 # the name of the driver class for mysql
                jdbc_driver_class => "com.mysql.jdbc.Driver"
                 # 数据库重连尝试次数
                connection_retry_attempts => "3"
                 # 判断数据库连接是否可用，默认false不开启
                jdbc_validate_connection => "true"
                 # 数据库连接可用校验超时时间，默认3600S
                jdbc_validation_timeout => "3600"
                 # 开启分页查询（默认false不开启）；
                jdbc_paging_enabled => "true"
                 # 单次分页查询条数（默认100000,若字段较多且更新频率较高，建议调低此值）；
                jdbc_page_size => "30"
                 # statement为查询数据sql，如果sql较复杂，建议配通过statement_filepath配置sql文件的存放路径；
                 # sql_last_value为内置的变量，存放上次查询结果中最后一条数据tracking_column的值，此处即为update_time；
                 # statement_filepath => "mysql/jdbc.sql"
                statement => "SELECT id,name,avatar,department,hospital,expertise,intro,update_time,status FROM scmy_doctors where `status`=0 and update_time>= :sql_last_value order by id desc"
                 # 是否将字段名转换为小写，默认true（如果有数据序列化、反序列化需求，建议改为false）；
                lowercase_column_names => false
                 # Value can be any of: fatal,error,warn,info,debug，默认info；
                sql_log_level => warn
                 #
                 # 是否记录上次执行结果，true表示会将上次执行结果的tracking_column字段的值保存到last_run_metadata_path指定的文件中；
                record_last_run => true
                 # 需要记录查询结果某字段的值时，此字段为true，否则默认tracking_column为timestamp的值；
                use_column_value => true
                 # 需要记录的字段，用于增量同步，需是数据库字段
                tracking_column => "update_time"
                 # Value can be any of: numeric,timestamp，Default value is "numeric"
                tracking_column_type => timestamp
                 # record_last_run上次数据存放位置；
                last_run_metadata_path => "/usr/share/logstash/mysql/doc_id.txt"
                 # 是否清除last_run_metadata_path的记录，需要增量同步时此字段必须为false；
                clean_run => false
                 #
                 # 设置监听间隔 各字段含义（从左至右）分、时、天、月、年，全为*默认含义为每分钟都更新
                schedule => "*/5 * * * *" #每5分钟更新一次
        }
 
 
}
 
filter {
        json {
                source => "message"
                remove_field => ["message"]
        }
         mutate{
             gsub => [ "content", "<script(.*?)</script>", "" ]
        }
         mutate{
             gsub => [ "content", "<iframe(.*?)</iframe>", "" ]
        }
         mutate{
             gsub => [ "content", "<style(.*?)</style>", "" ]
        }
         mutate{
             gsub => [ "content", "<(.*?)>", "" ]
        }
         mutate{
             gsub => [ "content", "&nbsp;", "" ]
        }
}
output {
        stdout {
                codec => json_lines
        }
        if[type]=="articles"{
            elasticsearch {
                 # host => "127.0.0.1"
                 # port => "9200"
                 # 配置ES集群地址
                hosts => "192.168.0.6:9200"
                 # 索引名字，必须小写
                index => "art_index"
                 # type名(7.X已弃用)
                 # document_type => "articles_type"
                 # 数据唯一索引（建议使用数据库id）
                document_id => "%{id}"
                manage_template =>false
                template_overwrite => true
                template => "/usr/share/logstash/template/art-ik.json"
           }
       }
        if[type]=="doc"{
            elasticsearch {
                 # host => "127.0.0.1"
                 # port => "9200"
                 # 配置ES集群地址
                hosts => "192.168.0.6:9200"
                 # 索引名字，必须小写
                index => "doc_index"
                 # type名(7.X已弃用)
                 # document_type => "articles_type"
                 # 数据唯一索引（建议使用数据库id）
                document_id => "%{id}"
                manage_template =>false
                template_overwrite => true
                template => "/usr/share/logstash/template/doc-ik.json"
           }
       }
}
 
# 索引模板配置 art-ik.json，doc-ik.json 放入template文件夹
#art-ik.json
{
    "order":3, 
    "version": 70001,
    "template": "art_*", #这里需要注意跟logstash.conf outPut的es配置中index匹配，如art_index，以art_开头的索引都会匹配上
    "settings": {
        "index": {
            "refresh_interval": "5s"
        },
        "analysis": {  #自定义了分词器，目的是为了去除文章内容html标签
            "analyzer": {
                "ik_html": {
                    "type": "custom",
                    "char_filter": [
                        "html_strip"
                    ],
                    "tokenizer": "ik_max_word",
                    "filter": [
                        "lowercase",
                        "stop",
                        "snowball"
                    ]
                }
            }
        }
    },
    "mappings": {
        "dynamic_templates": [
            {
                "message_field": {
                    "path_match": "message",
                    "mapping": {
                        "norms": false,
                        "type": "text"
                    },
                    "match_mapping_type": "string"
                }
            },
            {
                "string_fields": {
                    "mapping": {
                        "norms": false,
                        "type": "text",
                        "analyzer": "ik_html",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "match_mapping_type": "string",
                    "match": "*"
                }
            }
        ],
        "properties": {
            "@timestamp": {
                "type": "date"
            },
            "geoip": {
                "dynamic": true,
                "properties": {
                    "ip": {
                        "type": "ip"
                    },
                    "latitude": {
                        "type": "half_float"
                    },
                    "location": {
                        "type": "geo_point"
                    },
                    "longitude": {
                        "type": "half_float"
                    },
                    "id": {
                        "type": "long"
                    },
                    "title": {
                        "norms": false,
                        "analyzer": "ik_max_word",
                        "type": "text",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "create_time": {
                        "type": "date"
                    },
                    "content": {
                        "norms": false,
                        "analyzer": "ik_html",
                        "type": "text",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "update_time": {
                        "type": "date"
                    },
                    "status": {
                        "type": "integer"
                    }
                }
            },
            "@version": {
                "type": "keyword"
            }
        }
    }
}
 
#doc_index.json
{
    "order":2,
    "version": 70001,
    "template": "doc_*", #同理
    "settings": {
        "index": {
            "refresh_interval": "5s"
        }
    },
    "mappings": {
        "dynamic_templates": [
            {
                "message_field": {
                    "path_match": "message",
                    "mapping": {
                        "norms": false,
                        "type": "text"
                    },
                    "match_mapping_type": "string"
                }
            },
            {
                "string_fields": {
                    "mapping": {
                        "norms": false,
                        "type": "text",
                        "analyzer": "ik_max_word",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "match_mapping_type": "string",
                    "match": "*"
                }
            }
        ],
        "properties": {
            "@timestamp": {
                "type": "date"
            },
            "geoip": {
                "dynamic": true,
                "properties": {
                    "ip": {
                        "type": "ip"
                    },
                    "latitude": {
                        "type": "half_float"
                    },
                    "location": {
                        "type": "geo_point"
                    },
                    "longitude": {
                        "type": "half_float"
                    },
                    "id": {
                        "type": "long"
                    },
                    "name": {
                        "norms": false,
                        "analyzer": "ik_max_word",
                        "type": "text",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "avatar": {
                        "type": "text",
                        "index": false
                    },
                    "department": {
                        "norms": false,
                        "analyzer": "ik_max_word",
                        "type": "text",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "hospital": {
                        "norms": false,
                        "analyzer": "ik_max_word",
                        "type": "text",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "expertise": {
                        "norms": false,
                        "analyzer": "ik_max_word",
                        "type": "text",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "intro": {
                        "norms": false,
                        "analyzer": "ik_max_word",
                        "type": "text",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "update_time": {
                        "type": "date"
                    },
                    "status": {
                        "type": "integer"
                    }
                }
            },
            "@version": {
                "type": "keyword"
            }
        }
    }
}


# 创建新容器并挂载目录
docker run -di --name=logstash \
-e LS_JAVA_OPTS="-Xms512m -Xmx512m" \
-v /data/docker/logstash/config:/usr/share/logstash/config \
-v /data/docker/logstash/mysql:/usr/share/logstash/mysql \
-v /data/docker/logstash/template:/usr/share/logstash/template \
-v /data/docker/logstash/pipeline:/usr/share/logstash/pipeline logstash:7.6.2
 
#注意：创建容器这一步如果出现我下面的问题，两个索引模板需要先创建，再创建容器。
 
#自定义的模板没有生效，知道问题的大神希望能给解答一下，谢谢....
#生成art_index模板
PUT _template/art
{
    "order":2,
    "version": 70001,
    "template": "art_*",
    "settings": {
        "index": {
            "refresh_interval": "5s"
        },
        "analysis": {
            "analyzer": {
                "ik_html": {
                    "type": "custom",
                    "char_filter": [
                        "html_strip"
                    ],
                    "tokenizer": "ik_max_word",
                    "filter": [
                        "lowercase",
                        "stop",
                        "snowball"
                    ]
                }
            }
        }
    },
    "mappings": {
        "dynamic_templates": [
            {
                "message_field": {
                    "path_match": "message",
                    "mapping": {
                        "norms": false,
                        "type": "text"
                    },
                    "match_mapping_type": "string"
                }
            },
            {
                "string_fields": {
                    "mapping": {
                        "norms": false,
                        "type": "text",
                        "analyzer": "ik_html",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "match_mapping_type": "string",
                    "match": "*"
                }
            }
        ],
        "properties": {
            "@timestamp": {
                "type": "date"
            },
            "geoip": {
                "dynamic": true,
                "properties": {
                    "ip": {
                        "type": "ip"
                    },
                    "latitude": {
                        "type": "half_float"
                    },
                    "location": {
                        "type": "geo_point"
                    },
                    "longitude": {
                        "type": "half_float"
                    },
                    "id": {
                        "type": "long"
                    },
                    "title": {
                        "norms": false,
                        "analyzer": "ik_max_word",
                        "type": "text",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "create_time": {
                        "type": "date"
                    },
                    "content": {
                        "norms": false,
                        "analyzer": "ik_html",
                        "type": "text",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "update_time": {
                        "type": "date"
                    },
                    "status": {
                        "type": "integer"
                    }
                }
            },
            "@version": {
                "type": "keyword"
            }
        }
    }
}
 
#生成doc_index模板
PUT _template/doc
{
    "order":2,
    "version": 70001,
    "template": "doc_*", 
    "settings": {
        "index": {
            "refresh_interval": "5s"
        }
    },
    "mappings": {
        "dynamic_templates": [
            {
                "message_field": {
                    "path_match": "message",
                    "mapping": {
                        "norms": false,
                        "type": "text"
                    },
                    "match_mapping_type": "string"
                }
            },
            {
                "string_fields": {
                    "mapping": {
                        "norms": false,
                        "type": "text",
                        "analyzer": "ik_max_word",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "match_mapping_type": "string",
                    "match": "*"
                }
            }
        ],
        "properties": {
            "@timestamp": {
                "type": "date"
            },
            "geoip": {
                "dynamic": true,
                "properties": {
                    "ip": {
                        "type": "ip"
                    },
                    "latitude": {
                        "type": "half_float"
                    },
                    "location": {
                        "type": "geo_point"
                    },
                    "longitude": {
                        "type": "half_float"
                    },
                    "id": {
                        "type": "long"
                    },
                    "name": {
                        "norms": false,
                        "analyzer": "ik_max_word",
                        "type": "text",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "avatar": {
                        "type": "text",
                        "index": false
                    },
                    "department": {
                        "norms": false,
                        "analyzer": "ik_max_word",
                        "type": "text",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "hospital": {
                        "norms": false,
                        "analyzer": "ik_max_word",
                        "type": "text",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "expertise": {
                        "norms": false,
                        "analyzer": "ik_max_word",
                        "type": "text",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "intro": {
                        "norms": false,
                        "analyzer": "ik_max_word",
                        "type": "text",
                        "fields": {
                            "keyword": {
                                "ignore_above": 256,
                                "type": "keyword"
                            }
                        }
                    },
                    "update_time": {
                        "type": "date"
                    },
                    "status": {
                        "type": "integer"
                    }
                }
            },
            "@version": {
                "type": "keyword"
            }
        }
    }