# 环境说明
- k8s版本：1.20.0
- elfk版本：7.6.2
- elastalert：3.0.0
主要功能是通过ELFK收集业务日志进行处理，然后elastalert根据日志中的关键字进行匹配，将相对应的日志通过邮件或其它方式告警通知
# ELFK部署
博客之前有部署过ELFK可以参考，也可以直接使用以下的
## ES集群部署
```bash
[root@k8s01 elk]# vim es-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: es
namespace: tools-env
spec:
serviceName: elasticsearch
replicas: 3
selector:
matchLabels:
app: elasticsearch
template:
metadata:
labels:
app: elasticsearch
spec:
initContainers:
- name: increase-vm-max-map
image: busybox
imagePullPolicy: IfNotPresent
command: ["sysctl", "-w", "vm.max_map_count=262144"]
securityContext:
privileged: true
- name: increase-fd-ulimit
image: busybox
imagePullPolicy: IfNotPresent
command: ["sh", "-c", "ulimit -n 65536"]
securityContext:
privileged: true
containers:
- name: elasticsearch
image: elasticsearch:7.6.2
imagePullPolicy: IfNotPresent
ports:
- name: rest
containerPort: 9200
- name: inter
containerPort: 9300
resources:
limits:
cpu: 1000m
requests:
cpu: 1000m
volumeMounts:
- name: data
mountPath: /usr/share/elasticsearch/data
- name: localtime
mountPath: /etc/localtime
env:
- name: cluster.name
value: elasticsearch
- name: node.name
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: cluster.initial_master_nodes
value: "es-0,es-1,es-2"
- name: discovery.zen.minimum_master_nodes
value: "2"
- name: discovery.seed_hosts
value: "elasticsearch"
- name: ES_JAVA_OPTS
value: "-Xms512m -Xmx512m"
- name: network.host
value: "0.0.0.0"
volumes:
- name: localtime
hostPath:
path: /etc/localtime
volumeClaimTemplates:
- metadata:
name: data
labels:
app: elasticsearch
spec:
accessModes: [ "ReadWriteOnce" ]
storageClassName: managed-nfs-storage
resources:
requests:
storage: 50Gi
---
kind: Service
apiVersion: v1
metadata:
name: elasticsearch
namespace: tools-env
labels:
app: elasticsearch
spec:
selector:
app: elasticsearch
clusterIP: None
ports:
- port: 9200
name: rest
- port: 9300
name: inter-node
---
kind: Service
apiVersion: v1
metadata:
name: elasticsearch-client
namespace: tools-env
labels:
app: elasticsearch
spec:
selector:
app: elasticsearch
ports:
- port: 9200
name: rest
nodePort: 30920
- port: 9300
name: inter-node
nodePort: 30930
type: NodePort
[root@k8s01 elk]# kubectl apply -f es.yaml
statefulset.apps/es created
service/elasticsearch created
service/elasticsearch-client created
```
## Logstash部署
```bash
[root@k8s01 elk]# vim logstash-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: logstash-config
namespace: tools-env
labels:
app: logstash
data:
logstash.conf: |-
input {
beats {
host => "0.0.0.0"
port => 5044
}
}
output {
if [filetype] == "gts-center" {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "prod-gts-center_%{+YYYY-MM-dd}"
}
}
...
if [filetype] == "promotion-center" {
elasticsearch {
hosts => ["elasticsearch:9200"]
index => "prod-promotion-center_%{+YYYY-MM-dd}"
}
}
}
[root@k8s01 elk]# vim logstash.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: logstash
namespace: tools-env
spec:
replicas: 1
selector:
matchLabels:
app: logstash
template:
metadata:
labels:
app: logstash
spec:
containers:
- name: logstash
image: logstash:7.6.2
imagePullPolicy: IfNotPresent
volumeMounts:
- name: config
mountPath: /usr/share/logstash/pipeline/logstash.conf
subPath: logstash.conf
command:
- "/bin/sh"
- "-c"
- "/usr/share/logstash/bin/logstash -f /usr/share/logstash/pipeline/logstash.conf"
volumes:
- name: config
configMap:
name: logstash-config
---
apiVersion: v1
kind: Service
metadata:
labels:
app: logstash
name: logstash
namespace: tools-env
spec:
ports:
- port: 5044
targetPort: 5044
nodePort: 30504
selector:
app: logstash
type: NodePort
[root@k8s01 elk]# kubectl apply -f logstash-config.yaml
configmap/logstash-config created
[root@k8s01 elk]# kubectl apply -f logstash.yaml
deployment.apps/logstash created
service/logstash created
[root@k8s01 elk]# kubectl get -n tools-env po|grep logstash
logstash-679cb9c998-hpt6w 1/1 Running 1 25s
```
## Kibana部署
```bash
[root@k8s01 elk]# vim kibana-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: kibana-config
namespace: tools-env
labels:
app: kibana
data:
kibana.yml: |-
server.name: kibana
server.host: "0"
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
xpack.monitoring.ui.container.elasticsearch.enabled: true
[root@k8s01 elk]# vim kibana.yaml
apiVersion: v1
kind: Service
metadata:
name: kibana
namespace: tools-env
labels:
app: kibana
spec:
ports:
- port: 5601
nodePort: 30561
type: NodePort
selector:
app: kibana
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: kibana
namespace: tools-env
labels:
app: kibana
spec:
selector:
matchLabels:
app: kibana
template:
metadata:
labels:
app: kibana
spec:
containers:
- name: kibana
image: kibana:7.6.2
imagePullPolicy: IfNotPresent
env:
- name: ELASTICSEARCH_HOSTS
value: http://elasticsearch:9200
ports:
- containerPort: 5601
volumeMounts:
- name: config
mountPath: /usr/share/kibana/config/kibana.yml
subPath: kibana.yml
- name: localtime
mountPath: /etc/localtime
volumes:
- name: config
configMap:
name: kibana-config
- name: localtime
hostPath:
path: /usr/share/zoneinfo/Asia/Shanghai
[root@k8s01 elk]# kubectl apply -f kibana-config.yaml
configmap/kibana-config created
[root@k8s01 elk]# kubectl apply -f kibana.yaml
service/kibana created
deployment.apps/kibana created
```
## Filebeat部署
由于业务日志通过pvc持久化到NAS上，本文就直接采用rpm安装并收集对应NAS挂载目录日志，也可以使用DaemeSet的部署模式直接收集业务pod的日志，就看实际需求了。这里采用rpm安装
```bash
#下载filebeat的rpm文件
[root@elk ~]# wget https://repo.huaweicloud.com/filebeat/7.6.2/filebeat-7.6.2-x86_64.rpm
#安装filebeat
[root@elk ~]# rpm -ivh filebeat-7.6.2-x86_64.rpm
#修改配置文件（根据实际情况）
[root@elk ~]# cat /etc/filebeat/filebeat.yml
filebeat.inputs:
- type: log
enabled: true
backoff: "1s"
tail_files: false
paths:
- /home/logs/*.log #这里根据实际路径填写，可以多个path
fields:
filetype: crm-center
fields_under_root: true
multiline.pattern: '^\['
multiline.negate: true
multiline.match: after
setup.kibana:
host: "x.x.x.x:30561" #kibana地址
output.logstash:
enabled: true
hosts: ["x.x.x.x:30504"] #logstash地址
```
# Elastalert部署
## 创建secret
用作邮件告警的账号及密码
```bash
[root@elk ~]# vim email_auth.yaml
user: monitor@xxx.cn
password: 123456
#根据email_auth.yaml创建 secret
[root@k8s01 elk]# kubectl create secret generic smtp-auth --from-file=email_auth.yaml -n tools-env
secret/smtp-auth created
```
## 创建configmap
elastalert的配置文件、告警规则及邮箱账号密码
```bash
[root@k8s01 elk]# vim elastalert-config.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: elastalert-config
namespace: tools-env
data:
prod.yaml: |
es_host: elasticsearch
es_port: 9200
name: prod-server-rules
type: frequency
query_key: #不进行重复提醒的字段
- message
aggregation: #聚合1分钟内的结果，合并在一起发送
minutes: 1
realert: #同一规则的两次警报之间的最短时间
minutes: 2
#指数级扩大realert时间，中间如果有报警，则按照5->10->20->40->60不断增大报警时间到制定的最大时间，如果之后报警减少，则会慢慢恢复原始realert 时间
exponential_realert:
hours: 1
index: test-log* # 匹配索引，支持正则
num_events: 1 # 与规则匹配的日志出现次数
timeframe: # 在timeframe时间内出现num_events次与规则匹配的日志，将会触发报警
minutes: 1
#过滤规则
#根据报警的内，将相同的报警按照name来聚合
aggregation_key: name
filter:
- query:
query_string:
query: "message: ERRORMESSAGE"
alert:
- "email"
- "dingtalk"
email_format: html
alert_text_type: alert_text_only
#标题
alert_subject: "生产环境日志告警通知"
#主题模板
alert_text: "<br><a href='https://kibana.xxx.cn/app/kibana'target='_blank'style='padding: 8px 16px;background-color: #46bc99;text-decoration:none;color:white;border-radius: 5px;'>立刻前往Kibana查看</a><br>
<table>
<tr><td style='padding:5px;text-align: right;font-weight: bold;border-radius: 5px;background-color: #eef;'>告警时间</td>
<td style='padding:5px;border-radius: 5px;background-color: #eef;'>{}</td></tr>
<tr><td style='padding:5px;text-align: right;font-weight: bold;border-radius: 5px;background-color: #eef;'>服务名称</td>
<td style='padding:5px;border-radius: 5px;background-color: #eef;'>{}</td></tr>
<tr><td style='padding:5px;text-align: right;font-weight: bold;border-radius: 5px;background-color: #eef;'>服务器ID</td>
<td style='padding:5px;border-radius: 5px;background-color: #eef;'>{}</td></tr>
<tr><td style='padding:5px;text-align: right;font-weight: bold;border-radius: 5px;background-color: #eef;'>索引名称</td>
<td style='padding:5px;border-radius: 5px;background-color: #eef;'>{}</td></tr>
<tr><td style='padding:5px;text-align: right;font-weight: bold;border-radius: 5px;background-color: #eef;'>告警详情</td>
<td style='padding:10px 5px;border-radius: 5px;background-color: #F8F9FA;'>{}</td></tr>
<tr><td style='padding:5px;text-align: right;font-weight: bold;border-radius: 5px;background-color: #eef;'>规则命中数</td>
<td style='padding:5px;border-radius: 5px;background-color: #eef;'>{}</td></tr>
<tr><td style='padding:5px;text-align: right;font-weight: bold;border-radius: 5px;background-color: #eef;'>匹配触发数</td>
<td style='padding:5px;border-radius: 5px;background-color: #eef;'>{}</td></tr>
</table>"
# 告警模板引用
alert_text_args:
- "@timestamp"
- filetype
- host.name
- _index
- message
- num_hits
- num_matches
- path
email:
- "xxx@xxx.com.cn"
- "monitor@xxx.cn"
smtp_host: mail.xxx.cn
smtp_port: 25
smtp_auth_file: /opt/elastalert/smtp_auth.yaml
from_addr: monitor@xxx.cn
dingtalk_webhook_url: "https://oapi.dingtalk.com/robot/send?access_token=3fe29834a80edxxxxxxxx802071xxx630e34e3"
dingtalk_webhook_secret: "SEC7a5a905exxxx1a19860bfcxxxx62c3e117b4c1b72" #加签
dingtalk_msgtype: text
config.json: |
{
"appName": "elastalert-server",
"port": 3030,
"wsport": 3333,
"elastalertPath": "/opt/elastalert",
"verbose": false,
"es_debug": false,
"debug": false,
"rulesPath": {
"relative": true,
"path": "/rules"
},
"templatesPath": {
"relative": true,
"path": "/rule_templates"
},
"es_host": "elasticsearch",
"es_port": 9200,
"writeback_index": "elastalert_status"
}
elastalert.yaml: |
rules_folder: rules
run_every:
seconds: 60
buffer_time:
minutes: 15
es_host: elasticsearch
es_port: 9200
use_ssl: False
verify_certs: False
writeback_index: elastalert_status
writeback_alias: elastalert_alerts
alert_time_limit:
days: 2
smtp_auth.yaml: |
user: op@xxxx.cn
password: 123456
[root@k8s01 elk]# kubectl apply -f elastalert-config.yaml
configmap/elastalert-config created
```
## 部署elastalert
```bash
[root@k8s01 elk]# vim elastalert.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: elastalert
name: elastalert
namespace: tools-env
spec:
replicas: 1
selector:
matchLabels:
app: elastalert
template:
metadata:
labels:
app: elastalert
name: elastalert
spec:
containers:
- env:
- name: TZ
value: Asia/Shanghai
image: brycehuang/elastalert-dingtalk:3.0.0-beta.1 #这个镜像是加了钉钉模块的
#image: bitsensor/elastalert:3.0.0-beta.1
imagePullPolicy: IfNotPresent
name: elastalert
ports:
- containerPort: 3030
name: tcp-3030
protocol: TCP
- containerPort: 3333
name: tcp-3333
protocol: TCP
resources:
limits:
cpu: '2'
requests:
cpu: '1'
volumeMounts:
- mountPath: /opt/elastalert/config.yaml
name: elastalert-config
subPath: elastalert.yaml
- mountPath: /opt/elastalert/smtp_auth.yaml
name: elastalert-config
subPath: smtp_auth.yaml
- mountPath: /opt/elastalert-server/config/config.json
name: elastalert-config
subPath: config.json
- mountPath: /opt/elastalert/rules/prod.yaml
name: elastalert-config
subPath: prod.yaml
dnsPolicy: ClusterFirst
restartPolicy: Always
volumes:
- configMap:
defaultMode: 420
name: elastalert-config
name: elastalert-config
---
apiVersion: v1
kind: Service
metadata:
name: elastalert
namespace: tools-env
spec:
ports:
- name: serverport
port: 3030
protocol: TCP
targetPort: 3030
- name: transport
port: 3333
protocol: TCP
targetPort: 3333
selector:
app: elastalert
[root@k8s01 elk]# kubectl apply -f elastalert.yaml
statefulset.apps/elastalert created
service/elastalert created
```
## 模拟生产ERROR触发告警
正常下，kibana收集到日志如下:（这里是模拟产生的日志）
直接开始模拟日志产生ERROR，手动echo一些错误到日志文件中，结果如下：
邮件收到告警
k8s部署ELFK elastalert告警通知
Copyright: 采用 知识共享署名4.0 国际许可协议进行许可
Links: https://www.youqiqi.cn/archives/k8s-bu-shu-elfkelastalert-gao-jing-tong-zhi
