📦 K8s
版本 v18.10
,Redis
版本 7.2
🏆 k8s
搭建 Redis
集群
需要先安装rook 博客: https://blog.tanc.fun:9999/archives/b951eef6-2c22-40a5-b854-a04341e7377f
工序和上一篇差不多 blog
: https://blog.tanc.fun:9999/archives/a72f8744-60bf-4e3f-81c8-caf651e79b17
本操作集群结构为三主三从
🍪 目录结构
conf
目录存放配置文件用于configmap
Headless.yaml
配置Headlees
和读写svc
configmap.sh
,通过conf
和sh
目录生成configmap
sc.yaml
,使用他来对接rook
(详情请见博客https://blog.tanc.fun:9999/archives/b951eef6-2c22-40a5-b854-a04341e7377f)sh
目录用于存放初始化和集群脚本StatefulSet.yml
主要生成pod
文件
[root@k8s-master redis]# tree
.
├── conf
│ ├── redis.conf
│ └── redis-replica.conf
├── configmap.sh
├── Headless.yaml
├── label_role.sh
├── pod-test.yaml
├── sc.yaml
├── secret.yml
├── sh
│ ├── init-container.sh
│ └── run.sh
└── StatefulSet.yml
🌟 Headless
部署一个 Headless
,便于集群使用域名通讯,创建了一读一写的 svc
,用于读写分离,后续需要给读/写 的 pod
打上标签
[root@k8s-master redis]# cat Headless.yaml
apiVersion: v1
kind: Service
metadata:
name: redis
namespace: mall-app
labels:
app: redis
spec:
selector:
app: redis
ports:
- name: redis
port: 6379
clusterIP: None
---
apiVersion: v1
kind: Service
metadata:
name: redis-read
namespace: mall-app
labels:
app: redis
spec:
selector:
role: cache-read
ports:
- name: redis-read
port: 6379
---
apiVersion: v1
kind: Service
metadata:
name: redis-write
namespace: mall-app
labels:
app: redis
spec:
selector:
role: cache-write
ports:
- name: redis-write
port: 6379
创建后查看
[root@k8s-master redis]# kubectl get svc -n mall-app -l app=redis
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
redis ClusterIP None <none> 6379/TCP 7h27m
redis-read ClusterIP 10.103.240.178 <none> 6379/TCP 7h27m
redis-write ClusterIP 10.98.78.151 <none> 6379/TCP 7h27m
🌟 创建 StorageClass
用于对接 rook
[root@k8s-master redis]# cat sc.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
labels:
app: rook-nfs
name: rook-nfs-cache
namespace: mall-app
parameters:
exportName: nfs-share-01
nfsServerName: rook-nfs
nfsServerNamespace: rook-nfs
provisioner: nfs.rook.io/rook-nfs-provisioner
reclaimPolicy: Delete
volumeBindingMode: Immediate
创建后查看
[root@k8s-master redis]# kubectl get sc | grep cache
rook-nfs-cache nfs.rook.io/rook-nfs-provisioner Delete Immediate false 6h41m
🌟 编写配置文件和脚本文件
如果需要修改
redis
的密码请修改init
脚本和run
脚本的REDIS_PASS
字段吧
1️⃣ redis配置
[root@k8s-master redis]# cat conf/redis.conf
# 绑定的 IP 地址,通常注释掉以允许所有 IP 访问
#bind 127.0.0.1
# 端口
port 6379
# 数据库数量
databases 16
# 日志级别
loglevel notice
#后台运行
demonize yes
# 持久化相关设置(示例)
#save 900 1
#save 300 10
save 60 10000
# RDB 文件名称
dbfilename dump.rdb
# RDB 文件保存路径
dir /data
#是否可压缩rdb文件
rdbcompression yes
#AOF
appendonly yes
appendfsync everysec
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
#RDB-AOF混合持久化
aof-use-rdb-preamble yes
# 密码设置(如果需要)
# 最大客户端连接数
maxclients 10000
# 超时时间(秒)
timeout 0
#集群
cluster-enabled yes
cluster-node-timeout 20000
cluster-port 16379
cluster-replica-validity-factor 0
cluster-migration-barrier 1
cluster-replica-no-failover no
cluster-allow-reads-when-down yes
cluster-require-full-coverage no
##主从
replica-read-only yes
replica-serve-stale-data no
repl-ping-replica-period 10
repl-timeout 60
repl-backlog-size 20mb
#验证
masterauth bxzxhygr.25
2️⃣ init-conatiner.sh(不需要)
init-conatiner.sh(不需要)
该脚本主要用于配置初始化,在 init
容器中进行
[root@k8s-master redis]# cat sh/init-container.sh
#!/bin/bash
set -ex
REDIS_PASS=bxzxhygr.25
name=$(hostname|cut -d - -f 2)
cp /mnt/config-map/redis.conf /mnt/conf.d/redis.conf
echo cluster-config-file /data/nodes-$name.conf >> /mnt/conf.d/redis.conf
echo requirepass $REDIS_PASS >> /mnt/conf.d/redis.conf
cat /mnt/conf.d/redis.conf
3️⃣ run.sh
init-container
脚本现集成与 run.sh
之中该脚本为启动创建集群操作
[root@k8s-master redis]# cat sh/run.sh
#!/bin/bash
set -ex
REDIS_PASS=bxzxhygr.25
name=$(hostname|cut -d - -f 2)
## 将后续配置添加:
# 1. 集群文件,集群中的每个集群文件名需要不一样
# 2. 日志文件(可选)
# 3. 密码文件 可以自行配置
echo cluster-config-file /data/nodes-$name.conf >> /usr/local/etc/redis/redis.conf
echo logfile /data/redis-$name.log >> /usr/local/etc/redis/redis.conf
echo requirepass $REDIS_PASS >> /usr/local/etc/redis/redis.conf
## 查看redis配置文件
cat /usr/local/etc/redis/redis.conf
## 启动redis
redis-server /usr/local/etc/redis/redis.conf
## 等待集群redis全部启动
sleep 10s
## 测试连接每个节点的
until redis-cli -h 127.0.0.1 -a $REDIS_PASS PING; do sleep 5;done
until redis-cli -h redis-1.redis.mall-app.svc.cluster.local -a $REDIS_PASS PING;do sleep 5;done
until redis-cli -h redis-2.redis.mall-app.svc.cluster.local -a $REDIS_PASS PING;do sleep 5;done
until redis-cli -h redis-3.redis.mall-app.svc.cluster.local -a $REDIS_PASS PING;do sleep 5;done
until redis-cli -h redis-4.redis.mall-app.svc.cluster.local -a $REDIS_PASS PING;do sleep 5;done
until redis-cli -h redis-5.redis.mall-app.svc.cluster.local -a $REDIS_PASS PING;do sleep 5;done
## 判断是否为第一个节点
if [ $name -eq 0 ];then
cluster_node_num=$(redis-cli -h 127.0.0.1 -a $REDIS_PASS cluster info | grep 'cluster_slots_assigned:'| awk '{print $2}')
## 先判断是否执行过添加集群操作,执行添加至集群
if [ $cluster_node_num -gt 0 ];then
echo "yes" | redis-cli -h 127.0.0.1 -a $REDIS_PASS --cluster create redis-0.redis.mall-app.svc.cluster.local:6379 redis-1.redis.mall-app.svc.cluster.local:6379 redis-2.redis.mall-app.svc.cluster.local:6379 redis-3.redis.mall-app.svc.cluster.local:6379 redis-4.redis.mall-app.svc.cluster.local:6379 redis-5.redis.mall-app.svc.cluster.local:6379 --cluster-replicas 1
fi
redis-cli -h 127.0.0.1 -a $REDIS_PASS cluster info
redis-cli -h 127.0.0.1 -a $REDIS_PASS cluster nodes
else
## 其他节点查看主从信息即可
redis-cli -h 127.0.0.1 -a $REDIS_PASS info replication
fi
## 查看redis日志
tail -f /data/redis-$name.log
4️⃣ 创建 configmap
我也已经写好脚本了
[root@k8s-master redis]# cat configmap.sh
#!/bin/bash
kubectl delete configmap -n mall-app redis-config
kubectl create configmap -n mall-app redis-config --from-file conf/redis.conf,sh/run.sh
创建后查看
[root@k8s-master redis]# kubectl describe -n mall-app configmaps redis-config
Name: redis-config
Namespace: mall-app
Labels: <none>
Annotations: <none>
Data
====
redis.conf:
----
# 绑定的 IP 地址,通常注释掉以允许所有 IP 访问
#bind 127.0.0.1
# 端口
port 6379
# 数据库数量
databases 16
# 日志级别
loglevel notice
# 持久化相关设置(示例)
#save 900 1
#save 300 10
save 60 10000
# RDB 文件名称
dbfilename dump.rdb
# RDB 文件保存路径
dir /data
#是否可压缩rdb文件
rdbcompression yes
#AOF
appendonly yes
appendfsync everysec
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
#RDB-AOF混合持久化
aof-use-rdb-preamble yes
# 密码设置(如果需要)
# 最大客户端连接数
maxclients 10000
# 超时时间(秒)
timeout 0
#集群
cluster-enabled yes
cluster-node-timeout 20000
cluster-port 16379
cluster-replica-validity-factor 0
cluster-migration-barrier 1
cluster-replica-no-failover no
cluster-allow-reads-when-down yes
cluster-require-full-coverage no
##主从
replica-read-only yes
replica-serve-stale-data no
repl-ping-replica-period 10
repl-timeout 60
repl-backlog-size 20mb
#验证
masterauth bxzxhygr.25
run.sh:
----
#!/bin/bash
sleep 20
REDIS_PASS=bxzxhygr.25
set -ex
name=$(hostname|cut -d - -f 2)
until redis-cli -h 127.0.0.1 -a $REDIS_PASS PING; do sleep 5;done
until redis-cli -h redis-1.redis.mall-app.svc.cluster.local -a $REDIS_PASS PING;do sleep 5;done
until redis-cli -h redis-2.redis.mall-app.svc.cluster.local -a $REDIS_PASS PING;do sleep 5;done
until redis-cli -h redis-3.redis.mall-app.svc.cluster.local -a $REDIS_PASS PING;do sleep 5;done
until redis-cli -h redis-4.redis.mall-app.svc.cluster.local -a $REDIS_PASS PING;do sleep 5;done
until redis-cli -h redis-5.redis.mall-app.svc.cluster.local -a $REDIS_PASS PING;do sleep 5;done
if [ $name -eq 0 ];then
echo "yes" | redis-cli -h 127.0.0.1 -a $REDIS_PASS --cluster create redis-0.redis.mall-app.svc.cluster.local:6379 redis-1.redis.mall-app.svc.cluster.local:6379 redis-2.redis.mall-app.svc.cluster.local:6379 redis-3.redis.mall-app.svc.cluster.local:6379 redis-4.redis.mall-app.svc.cluster.local:6379 redis-5.redis.mall-app.svc.cluster.local:6379 --cluster-replicas 1
redis-cli -h 127.0.0.1 -a $REDIS_PASS cluster info
redis-cli -h 127.0.0.1 -a $REDIS_PASS cluster nodes
else
redis-cli -h 127.0.0.1 -a $REDIS_PASS info replication
fi
tail -f /etc/shadow
init-container.sh:
----
#!/bin/bash
set -ex
REDIS_PASS=bxzxhygr.25
name=$(hostname|cut -d - -f 2)
cp /mnt/config-map/redis.conf /mnt/conf.d/redis.conf
echo cluster-config-file /data/nodes-$name.conf >> /mnt/conf.d/redis.conf
echo requirepass $REDIS_PASS >> /mnt/conf.d/redis.conf
cat /mnt/conf.d/redis.conf
Events: <none>
🌟 创建 Statefulset.yaml
直接在主容器中运行 run.c
脚本即可
[root@k8s-master redis]# cat StatefulSet.yml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: redis
namespace: mall-app
labels:
app: redis
spec:
replicas: 6
selector:
matchLabels:
app: redis
serviceName: redis
template:
metadata:
labels:
app: redis
spec:
containers:
- name: redis
image: docker.io/library/redis:7.2
command:
- sh
- -c
- |
ls /usr/local/configmap/
mkdir /usr/local/etc/redis/ -p
cat /usr/local/configmap/redis.conf > /usr/local/etc/redis/redis.conf
cat /usr/local/configmap/run.sh > /tmp/run.sh
ll /tmp/run.sh
chmod +x /tmp/run.sh
/tmp/run.sh
volumeMounts:
- name: host-time
mountPath: /etc/localtime
- name: cache
mountPath: /data
- name: config-map
mountPath: /usr/local/configmap/
volumes:
- name: host-time
hostPath:
path: /etc/localtime
type: ''
- name: config-map
configMap:
name: redis-config
volumeClaimTemplates:
- metadata:
name: cache
spec:
storageClassName: rook-nfs-cache
accessModes:
- ReadWriteMany
resources:
requests:
storage: 500Mi
1️⃣ init
容器分析(不需要)
init
容器分析(不需要)容器镜像可以选择其他的,volume
挂载了两个一个是 config-map
,一个则是 empty
,command
中执行的 init
脚本大致内容为从 config-map
中复制 redis.conf
,到 empty
盘中,以便后续 redis
服务使用
initContainers:
- name: init-redis
image: docker.io/library/redis:7.2
command:
- sh
- -c
- |
ls /mnt/config-map
cp /mnt/config-map/init-container.sh /mnt/conf.d
chmod +x /mnt/conf.d/init-container.sh
/mnt/conf.d/init-container.sh
rm -rf /mnt/conf.d/init-container.sh
volumeMounts:
- name: config
mountPath: /mnt/conf.d
- name: config-map
mountPath: /mnt/config-map
2️⃣ redis
边车容器(不需要)
redis
边车容器(不需要)redis
边车容器主要是用来执行创建集群工作
- name: redis-sidecar
image: docker.io/library/redis:7.2
command:
- sh
- -c
- |
ls /mnt/config-map
cp /mnt/config-map/run.sh /data
chmod +x /data/run.sh
/data/run.sh
rm -rf /data/run.sh
volumeMounts:
- name: config-map
mountPath: /mnt/config-map
创建 statefulset
后查看
⚠️ 请注意这里虽然都显示running了还是需要进入容器或者查看日志是否有报错的
[root@k8s-master redis]# kubectl get pod -n mall-app -l app=redis
NAME READY STATUS RESTARTS AGE
redis-0 2/2 Running 0 56m
redis-1 2/2 Running 0 56m
redis-2 2/2 Running 0 56m
redis-3 2/2 Running 0 56m
redis-4 2/2 Running 0 55m
redis-5 2/2 Running 0 55m
🌟 给pod打上读写分离标签
[root@k8s-master redis]# cat label_role.sh
kubectl label pod -n mall-app redis-0 role=cache-write --overwrite
kubectl label pod -n mall-app redis-1 role=cache-write --overwrite
kubectl label pod -n mall-app redis-2 role=cache-write --overwrite
kubectl label pod -n mall-app redis-3 role=cache-read --overwrite
kubectl label pod -n mall-app redis-4 role=cache-read --overwrite
kubectl label pod -n mall-app redis-5 role=cache-read --overwrite
执行后查看 endpoint
[root@k8s-master redis]# kubectl get endpoints -n mall-app
NAME ENDPOINTS AGE
mariadb 10.244.0.105:3306,10.244.1.62:3306 31h
mariadb-read 10.244.0.105:3306,10.244.1.62:3306 31h
redis 10.244.0.134:6379,10.244.0.135:6379,10.244.0.136:6379 + 3 more... 7h39m
redis-read 10.244.0.136:6379,10.244.1.102:6379,10.244.1.103:6379 7h39m
redis-write 10.244.0.134:6379,10.244.0.135:6379,10.244.1.101:6379 7h39m
⭐️ 测试
1️⃣ 使用 read-cli
连接 write
##查看读的svc
[root@k8s-master ~]# kubectl get svc -n mall-app | grep redis-write
redis-write ClusterIP 10.98.78.151 <none> 6379/TCP 7h43m
###连接测试
[root@k8s-master ~]# redis-cli -h 10.98.78.151 -a bxzxhygr.25
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.98.78.151:6379> info replication
# Replication
role:master
....
slave0:ip=10.244.0.136,port=6379,state=online,offset=5292,lag=0
....
10.98.78.151:6379> exit
[root@k8s-master ~]# redis-cli -h 10.98.78.151 -a bxzxhygr.25
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.98.78.151:6379> info replication
# Replication
role:master
....
slave0:ip=10.244.1.103,port=6379,state=online,offset=5292,lag=1
....
##顺带查看一下redis cluster的信息
10.98.78.151:6379> cluster nodes
484e7e5f948b27d1f3a30f7c8df5b72308f5b04f 10.244.1.103:6379@16379 slave c2f47e716dc9050c53cd4355aa6e625d9c274457 0 1729173171108 1 connected
f0005a40d694e3db5d62b189da77a53a8d80155a 10.244.0.134:6379@16379 master - 0 1729173169083 2 connected 5461-10922
c42d218202c06300fff21fc127bde6be1bb6e122 10.244.0.135:6379@16379 master - 0 1729173172117 3 connected 10923-16383
76f0a73e5a6aeabfa38857121dfcc24b1fd78adf 10.244.0.136:6379@16379 slave f0005a40d694e3db5d62b189da77a53a8d80155a 0 1729173171000 2 connected
c2f47e716dc9050c53cd4355aa6e625d9c274457 10.244.1.101:6379@16379 myself,master - 0 1729173167000 1 connected 0-5460
333cc153bf3c73e92c7e395158bef1b06ae6b0fe 10.244.1.102:6379@16379 slave c42d218202c06300fff21fc127bde6be1bb6e122 0 1729173170098 3 connected
10.98.78.151:6379> cluster info
cluster_state:ok
cluster_slots_assigned:16384
cluster_slots_ok:16384
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3 ##集群中有3个主机
cluster_current_epoch:6
cluster_my_epoch:1
cluster_stats_messages_ping_sent:3938
cluster_stats_messages_pong_sent:4003
cluster_stats_messages_sent:7941
cluster_stats_messages_ping_received:3998
cluster_stats_messages_pong_received:3938
cluster_stats_messages_meet_received:5
cluster_stats_messages_received:7941
total_cluster_links_buffer_limit_exceeded:0
可以看见两次连接 pod
不一样,svc
自动帮我做了负载均衡
2️⃣ 使用 read-cli
连接 read
[root@k8s-master ~]# kubectl get svc -n mall-app | grep redis-read
redis-read ClusterIP 10.103.240.178 <none> 6379/TCP 7h49m
[root@k8s-master ~]# redis-cli -h 10.103.240.178 -a bxzxhygr.25
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.103.240.178:6379> info replication
# Replication
role:slave
master_host:10.244.1.101
master_port:6379
master_link_status:up
master_last_io_seconds_ago:8
master_sync_in_progress:0
slave_read_repl_offset:5586
slave_repl_offset:5586
slave_priority:100
slave_read_only:1
replica_announced:1
connected_slaves:0
master_failover_state:no-failover
master_replid:d5dfe86ce6ea7f7178c7450cbfa47d8dfd4d1ddd
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:5586
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:20971520
repl_backlog_first_byte_offset:1
repl_backlog_histlen:5586
10.103.240.178:6379> exit
[root@k8s-master ~]# redis-cli -h 10.103.240.178 -a bxzxhygr.25
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.103.240.178:6379> info replication
# Replication
role:slave
master_host:10.244.0.135
master_port:6379
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_read_repl_offset:5614
slave_repl_offset:5614
slave_priority:100
slave_read_only:1
replica_announced:1
connected_slaves:0
master_failover_state:no-failover
master_replid:a6153da96f60660000da4fc4adcb397d1b09db38
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:5614
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:20971520
repl_backlog_first_byte_offset:15
repl_backlog_histlen:5600
10.103.240.178:6379> exit
[root@k8s-master ~]# redis-cli -h 10.103.240.178 -a bxzxhygr.25
Warning: Using a password with '-a' or '-u' option on the command line interface may not be safe.
10.103.240.178:6379> info replication
# Replication
role:slave
master_host:10.244.0.134
master_port:6379
master_link_status:up
master_last_io_seconds_ago:6
master_sync_in_progress:0
slave_read_repl_offset:5600
slave_repl_offset:5600
slave_priority:100
slave_read_only:1
replica_announced:1
connected_slaves:0
master_failover_state:no-failover
master_replid:b6ad793a99aeb29749b6fea27506ac7a2f405ed3
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:5600
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:20971520
repl_backlog_first_byte_offset:1
repl_backlog_histlen:5600