安装 filebeat 7.17.10
创建yum源文件
[elastic-7.x]
name=Elastic repository for 7.x packages
baseurl=https://artifacts.elastic.co/packages/7.x/yum
gpgcheck=0
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
yum安装
yum install filebeat
启动并设置开机启动
service filebeat start
systemctl enable filebeat
启用kafka模块
# 查看支持的模块
filebeat modules list
filebeat modules enable kafka
配置文件
# logging.level: debug
filebeat.inputs:
- type: log
id: 192.168.240.190
enabled: true
paths:
- /data/gamelog/*.log
filebeat.config.modules:
path: ${path.config}/modules.d/*.yml
reload.enabled: false
reload.period: 10s
output.kafka:
enabled: true
hosts: ["192.168.240.121:9092"]
topic: xm2-game-log
keep_alive: 10s
timeout: 5
compression: gzip # 压缩
max_message_bytes: 1000000 # Event最大字节数。默认1000000。应小于等于kafka broker message.max.bytes值
required_acks: 1 # kafka ack等级
worker: 1 # kafka output的最大并发数
bulk_max_size: 2048 # 单次发往kafka的最大事件数
logging.to_files: true # 输出所有日志到file,默认true, 达到日志文件大小限制时,日志文件会自动限制替换
codec.format:
string: '%{[message]}'
重启即可生效
service filebeat restart
创建测试数据
vim /data/gamelog/test.log
{
"logtime": "2023-01-01 10:00:00",
"ip": "127.0.0.1",
"game_id": "xm2",
"package_id": "",
"platform": "117",
"group": "11700003",
"account": "local_ben",
"role_id": "96772055629825",
"role_name": "S56.Demi-Leigh",
"log_type": "RoleRegisterLog",
"role_career": ""
}