Centos 7 部署Kafka集群

系统初始化

#安装依赖
yum -y install java-1.7.0-openjdk* vim telnet lrzsz rsync  
#磁盘格式化
mkfs.ext4  /dev/vdb  
mkdir /data/ && mount /dev/vdb /data && df -h  
#开机自动挂载
data_blkid=`blkid |grep vdb |awk '{print $2}'|sed s/\"//g`  
echo "$data_blkid /data              ext4    defaults        0 0" >> /etc/fstab  

下载安装kafka和zookeeper

cd /opt/  
wget -q http://mirrors.shu.edu.cn/apache/kafka/0.10.2.1/kafka_2.10-0.10.2.1.tgz  
wget -q http://mirrors.shu.edu.cn/apache/zookeeper/stable/zookeeper-3.4.10.tar.gz  
tar zxf kafka_2.10-0.10.2.1.tgz  -C /opt/  
tar zxf zookeeper-3.4.10.tar.gz -C /opt/  

配置zookeeper

cd /data  
mkdir -p {zookeeper,zookeeper/zkdata,zookeeper/zkdatalog}

cat > /opt/zookeeper-3.4.10/conf/zoo.cfg <<EOF  
tickTime=2000  
initLimit=10  
syncLimit=5  
dataDir=/data/zookeeper/zkdata  
dataLogDir=/data/zookeeper/zkdatalog  
clientPort=12181  
server.1=192.168.0.138:12888:13888  
server.2=192.168.0.174:12888:13888  
server.3=192.168.0.125:12888:13888  
EOF

#server 1
echo "1" > /data/zookeeper/zkdata/myid  
#server 2
echo "2" > /data/zookeeper/zkdata/myid  
#server 3 
echo "3" > /data/zookeeper/zkdata/myid

• 启动zookeeper

cd /opt/zookeeper-3.4.10/bin && ./zkServer.sh start  

配置kafka

mkdir -p /data/kafka/kafkalogs/

cat > /opt/kafka_2.10-0.10.2.1/config/server.properties <<EOF  
broker.id=2    #不同的节点id不一样  
port=19092  
host.name=192.168.0.125  
num.network.threads=3  
num.io.threads=8  
log.dirs=/data/kafka/kafkalogs/  
socket.send.buffer.bytes=102400  
socket.receive.buffer.bytes=102400  
socket.request.max.bytes=104857600  
num.partitions=1  
log.retention.hours=168  
message.max.byte=5242880  
default.replication.factor=2  
replica.fetch.max.bytes=5242880  
log.segment.bytes=1073741824  
log.retention.check.interval.ms=300000  
log.cleaner.enable=false  
zookeeper.connect=192.168.0.138:12181,192.168.0.174:12181,192.168.0.125:1218  
EOF

• 启动kafka

cd /opt/kafka_2.10-0.10.2.1/bin  
./kafka-server-start.sh -daemon ../config/server.properties

测试

./kafka-topics.sh --create --zookeeper 192.168.0.138:12181 --replication-factor 2 --partitions 1 --topic portals

./kafka-console-producer.sh --broker-list 192.168.0.138:19092 --topic portals

./kafka-console-consumer.sh --zookeeper localhost:12181 --topic portals --from-beginning