软件
产品
一、安装fluent-bit ,两种方法
方法一、docker
mkdir -p /data/fluent-bit/etc
cat > /data/fluent-bit/etc/fluent-bit.conf << 'EOF'
[SERVICE]
flush 1
daemon Off
log_level info
#parsers_file parsers.conf
#plugins_file plugins.conf
http_server Off
http_listen 0.0.0.0
http_port 2020
storage.metrics on
[INPUT]
name tail
tag accessgate
ignore_older 2h
buffer_chunk_size 32k
buffer_max_size 64K
path /data/log/accessgate/*.access*.csv
db /tmp/accessgate.db
exclude_Path *.gz,*.zip
key message
[OUTPUT]
name es
host 192.168.11.100
port 9200
http_user elastic
http_passwd password
match accessgate
index sg-access
pipeline sg-access
EOF
cat > /data/fluent-bit/start.sh << 'EOF'
docker run -it \
--name fluent-bit \
--restart always \
-p 2020:2020 \
-v /etc/localtime:/etc/localtime \
-v /data/log/:/data/log/ \
-v `pwd`/etc/:/fluent-bit/etc/ \
fluent/fluent-bit:1.9.3
EOF
cd /data/fluent-bit/ && bash start.sh
方法二、 yum
1、准备tg-agent-bit安装源
cat > /etc/yum.repos.d/fluent-bit.repo << 'EOF'
[td-agent-bit]
name = TD Agent Bit
baseurl = https://packages.fluentbit.io/centos/7/$basearch/
gpgcheck=1
gpgkey=https://packages.fluentbit.io/fluentbit.key
enabled=1
EOF
2、通过yum安装 td -agent-bit
yum install td-agent-bit -y
systemctl start td-agent-bit && systemctl enable td-agent-bit #centos7
service td-agent-bit start && chkconfig td-agent-bit on #centos6
二、网关日志
2、fluent-bit代替filebeat
filebeat.inputs:
- type: log
enabled: true
paths:
- "/data/rio/log/accessgate/*.access*.csv"
fields:
type: "accessgate"
pipeline: "sg-access"
output.elasticsearch:
username: elastic
password: password
hosts:
- http://192.168.11.100:9200
worker: 2
bulk_max_size: 256
indices:
- index: "sg-access"
when.equals:
fields:
type: "accessgate"
2.2、fluent-bit配置
fluent-bit不支持elasticsearch 集群 多主机的配置
cat > /etc/td-agent-bit/td-agent-bit.conf << 'EOF'
[SERVICE]
flush 1
daemon Off
log_level info
#parsers_file parsers.conf
#plugins_file plugins.conf
http_server Off
http_listen 0.0.0.0
http_port 2020
storage.metrics on
[INPUT]
name tail
tag accessgate
ignore_older 2h
buffer_chunk_size 32k
buffer_max_size 64K
path /data/log/accessgate/*.access*.csv
db /tmp/accessgate.db
exclude_Path *.gz,*.zip
key message
[OUTPUT]
name es
host 192.168.11.100
port 9200
http_user elastic
http_passwd password
match accessgate
index sg-access
pipeline sg-access
EOF
三、 nginx 日志
[SERVICE]
flush 1
daemon Off
log_level info
parsers_file parsers.conf #引用parsers.conf文件
plugins_file plugins.conf
http_server Off
http_listen 0.0.0.0
http_port 2020
storage.metrics on
[INPUT]
name tail
tag nginxaccess
parser nginx #使用parsers.conf文件中的nginx parser.
ignore_Older 2h
buffer_Chunk_Size 32k
buffer_Max_Size 64K
path /data/nginx/log/access.log
db /tmp/nginx.db
exclude_Path *.gz,*.zip
key message
[OUTPUT]
name es
host 192.168.11.100
port 9200
http_user elastic
http_passwd password
match nginxaccess
index nginxaccess
parsers.conf
[PARSER]
Name nginx
Format regex
Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
验证结果如下
参考：https://blog.csdn.net/u012516914/article/details/106717302
fluent-bit.conf: |
[SERVICE]
Flush 1
Log_Level info
Daemon off
Parsers_File parsers.conf
HTTP_Server On
HTTP_Listen 0.0.0.0
HTTP_Port 2020
@INCLUDE input-kubernetes.conf
@INCLUDE filter-kubernetes.conf
@INCLUDE output-elasticsearch.conf
input-kubernetes.conf: |
[INPUT]
Name tail
Tag kube.*
Path /var/log/containers/*.log
Parser docker
DB /var/log/flb_kube.db
Mem_Buf_Limit 5MB
Skip_Long_Lines On
Refresh_Interval 10
[INPUT]
Name systemd
Tag host.*
Systemd_Filter _SYSTEMD_UNIT=kubelet.service
Path /var/log/journal
DB /var/log/flb_host.db
filter-kubernetes.conf: |
[FILTER]
Name kubernetes
Match kube.*
Kube_URL https://kubernetes.default.svc.cluster.local:443
Merge_Log On
K8S-Logging.Parser On
K8S-Logging.Exclude On
[FILTER]
Name kubernetes
Match host.*
Kube_URL https://kubernetes.default.svc.cluster.local:443
Merge_Log On
Use_Journal On
output-elasticsearch.conf: |
[OUTPUT]
Name es
Match *
Host ${FLUENT_ELASTICSEARCH_HOST}
Port ${FLUENT_ELASTICSEARCH_PORT}
Logstash_Format On
Retry_Limit False
parsers.conf: |
[PARSER]
Name apache
Format regex
Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name apache2
Format regex
Regex ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name apache_error
Format regex
Regex ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
[PARSER]
Name nginx
Format regex
Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name json
Format json
Time_Key time
Time_Format %d/%b/%Y:%H:%M:%S %z
[PARSER]
Name docker
Format json
Time_Key time
Time_Format %Y-%m-%dT%H:%M:%S.%L
Time_Keep On
# Command | Decoder | Field | Optional Action
# =============|==================|=================
Decode_Field_As escaped log
[PARSER]
Name syslog
Format regex
Regex ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
Time_Key time
Time_Format %b %d %H:%M:%S
https://docs.fluentbit.io/manual/ pipeline /outputs/elasticsearch
免责声明：本文系网络转载或改编，未找到原创作者，版权归原作者所有。如涉及版权，请联系删
技术文档
热门文章
155-2731-8020
