使用Docker来搭建Redis集群

发表于 | 分类于 ， |

注意：生产中，不使用。
（1）因为 Docker 容器直接通信不是直接通信的，都是转发的 Docker0 bridge 网桥，在数据量大的时候，会拉低 Redis性能。
（2）可能 Docker 服务经常挂掉，Redis 也就不可用了。一般一台服务器只装一台 Redis，用来最大化 Redis性能。
参考文献：
基于Docker搭建redis集群
docker 实现redis集群搭建

详细步骤

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
# 1. 创建Docker自定义网络
[root@localhost ~]# docker network create redis --subnet 172.22.0.0/16 --gateway 172.22.0.1
7fcd7743a831171236b7fabb468bfea0ac538ff4a7c13fa74d02043027ebab36
# 2. 查看网络列表
[root@localhost ~]# docker network ls
NETWORK ID          NAME                DRIVER              SCOPE
cdd8f5efd833        bridge              bridge              local
c7b56eb97499        example_default     bridge              local
b0e468be22b6        host                host                local
09be5ec40300        my_net              bridge              local
37f49d03f5d5        none                null                local
7fcd7743a831        redis               bridge              local # 我们创建的网络，名称：redis
# 3. 查看网络详情
[root@localhost ~]# docker network inspect redis
[
    {
        "Name": "redis",
        "Id": "7fcd7743a831171236b7fabb468bfea0ac538ff4a7c13fa74d02043027ebab36",
        "Created": "2020-06-11T17:55:13.633443866-07:00",
        "Scope": "local",
        "Driver": "bridge",
        "EnableIPv6": false,
        "IPAM": {
            "Driver": "default",
            "Options": {},
            "Config": [
                {
                    "Subnet": "172.22.0.0/16",
                    "Gateway": "172.22.0.1"
                }
            ]
        },
        "Internal": false,
        "Attachable": false,
        "Containers": {},
        "Options": {},
        "Labels": {}
    }
]
# 4. 循环创建6个容器，代码见文章下面的【代码】
[root@localhost ~]# for n in $(seq 1 6); \
> do \
> mkdir -p /app/docker/redis/cluster/node-${n}/conf
> cat > /app/docker/redis/cluster/node-${n}/conf/redis.conf << EOF
> port 6379
> cluster-enabled yes # 开启集群
> cluster-config-file nodes.conf # 集群配置文件名称
> cluster-node-timeout 5000 # Redis群集节点不可用的最长时间，毫秒
> cluster-announce-ip 172.22.0.1${n} # 注意 EOF不可用加单引号，双引号，否则这里会拼接字符串 '${n}'
> cluster-announce-port 6379 # 普通客户端通信端口（通常为6379）
> cluster-announce-bus-port 16379 # 群集总线端口（客户端端口+ 10000）
> appendonly yes
> EOF
>
> docker run --name redis-${n} -p 637${n}:6379 -p 1637${n}:16379 \
> -v /app/docker/redis/cluster/node-${n}/data:/data \ # 挂载目录
> -v /app/docker/redis/cluster/node-${n}/conf/redis.conf:/usr/local/etc/redis/redis.conf \ # 挂载配置文件
> --net redis --ip 172.22.0.1${n} -d redis redis-server /usr/local/etc/redis/redis.conf # 指定网络，指定IP，后天运行，指定镜像，指定配置文件启动
>
> done
dc086642a7fe2832ee297a8e74cb0a743cd7b71a42a8504724a431bbf22b866d
00eb966ff121f16bde383c3b0cb6ae1649d0fdbdb8276175881426e71204ef29
013904d59c0c4b0847d1c8a2d06bad5359e0ecb510b9b310ddceaa2105abf25a
6037e267c8cfaa43a4b33110ff5c856bbaad844a07aac702f1dfba2892a49858
431b926a1f8f97759d8b3d76729a88133dd0314b6f54065fdf5fbd538581be25
5d2dfa08a5b7de7b410766fcf1389d2bf7243aaf6d89766fb5366bb7cf74e108
# 5. 查看运行中的镜像
[root@localhost ~]# docker ps
CONTAINER ID        IMAGE               COMMAND                  CREATED             STATUS              PORTS                                              NAMES
5d2dfa08a5b7        redis               "docker-entrypoint..."   12 seconds ago      Up 11 seconds       0.0.0.0:6376->6379/tcp, 0.0.0.0:16376->16379/tcp   redis-6
431b926a1f8f        redis               "docker-entrypoint..."   12 seconds ago      Up 11 seconds       0.0.0.0:6375->6379/tcp, 0.0.0.0:16375->16379/tcp   redis-5
6037e267c8cf        redis               "docker-entrypoint..."   13 seconds ago      Up 12 seconds       0.0.0.0:6374->6379/tcp, 0.0.0.0:16374->16379/tcp   redis-4
013904d59c0c        redis               "docker-entrypoint..."   13 seconds ago      Up 12 seconds       0.0.0.0:6373->6379/tcp, 0.0.0.0:16373->16379/tcp   redis-3
00eb966ff121        redis               "docker-entrypoint..."   14 seconds ago      Up 13 seconds       0.0.0.0:6372->6379/tcp, 0.0.0.0:16372->16379/tcp   redis-2
dc086642a7fe        redis               "docker-entrypoint..."   15 seconds ago      Up 13 seconds       0.0.0.0:6371->6379/tcp, 0.0.0.0:16371->16379/tcp   redis-1
# 6. 进入1号容器
[root@localhost ~]# docker exec -it redis-1 bash
# 7. 创建集群，副本指定1，这里表示3主3从
root@dc086642a7fe:/data# redis-cli --cluster create 172.22.0.11:6379 172.22.0.12:6379 172.22.0.13:6379 \
> 172.22.0.14:6379 172.22.0.15:6379 172.22.0.16:6379 --cluster-replicas 1
>>> Performing hash slots allocation on 6 nodes...
Master[0] -> Slots 0 - 5460
Master[1] -> Slots 5461 - 10922
Master[2] -> Slots 10923 - 16383
Adding replica 172.22.0.15:6379 to 172.22.0.11:6379
Adding replica 172.22.0.16:6379 to 172.22.0.12:6379
Adding replica 172.22.0.14:6379 to 172.22.0.13:6379
M: c75c0d59d8749a60300f75d13f1caff15483297c 172.22.0.11:6379
   slots:[0-5460] (5461 slots) master
M: 51e77e699980423a34a6445adcb2de43b4d6022e 172.22.0.12:6379
   slots:[5461-10922] (5462 slots) master
M: 2716059d756823bdb745896d81b0d160263acd2f 172.22.0.13:6379
   slots:[10923-16383] (5461 slots) master
S: 6d538989252611041d8410fd90aabec0f237e00a 172.22.0.14:6379
   replicates 2716059d756823bdb745896d81b0d160263acd2f
S: 55e20690b236d9e2b00e91d53ac8919732d01597 172.22.0.15:6379
   replicates c75c0d59d8749a60300f75d13f1caff15483297c
S: 572ecc74c40e688e095310d5f3eb5c05c28aa354 172.22.0.16:6379
   replicates 51e77e699980423a34a6445adcb2de43b4d6022e
# 8. 输入 yes，确认设置
Can I set the above configuration? (type 'yes' to accept): yes
>>> Nodes configuration updated
>>> Assign a different config epoch to each node
>>> Sending CLUSTER MEET messages to join the cluster
Waiting for the cluster to join
.....
>>> Performing Cluster Check (using node 172.22.0.11:6379)
M: c75c0d59d8749a60300f75d13f1caff15483297c 172.22.0.11:6379
   slots:[0-5460] (5461 slots) master
   1 additional replica(s)
M: 51e77e699980423a34a6445adcb2de43b4d6022e 172.22.0.12:6379
   slots:[5461-10922] (5462 slots) master
   1 additional replica(s)
S: 55e20690b236d9e2b00e91d53ac8919732d01597 172.22.0.15:6379
   slots: (0 slots) slave
   replicates c75c0d59d8749a60300f75d13f1caff15483297c
M: 2716059d756823bdb745896d81b0d160263acd2f 172.22.0.13:6379
   slots:[10923-16383] (5461 slots) master
   1 additional replica(s)
S: 6d538989252611041d8410fd90aabec0f237e00a 172.22.0.14:6379
   slots: (0 slots) slave
   replicates 2716059d756823bdb745896d81b0d160263acd2f
S: 572ecc74c40e688e095310d5f3eb5c05c28aa354 172.22.0.16:6379
   slots: (0 slots) slave
   replicates 51e77e699980423a34a6445adcb2de43b4d6022e
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.
# 9. 以集群模式进入。注意：不加-c表示单机进入
root@dc086642a7fe:/data# redis-cli -c
# 10. 查看集群所有节点
127.0.0.1:6379> CLUSTER NODES
51e77e699980423a34a6445adcb2de43b4d6022e 172.22.0.12:6379@16379 master - 0 1591930968000 2 connected 5461-10922 # 主
c75c0d59d8749a60300f75d13f1caff15483297c 172.22.0.11:6379@16379 myself,master - 0 1591930967000 1 connected 0-5460 # 主，当前节点
55e20690b236d9e2b00e91d53ac8919732d01597 172.22.0.15:6379@16379 slave c75c0d59d8749a60300f75d13f1caff15483297c 0 1591930968855 5 connected # 从
2716059d756823bdb745896d81b0d160263acd2f 172.22.0.13:6379@16379 master - 0 1591930969000 3 connected 10923-16383 # 主
6d538989252611041d8410fd90aabec0f237e00a 172.22.0.14:6379@16379 slave 2716059d756823bdb745896d81b0d160263acd2f 0 1591930968000 4 connected # 从
572ecc74c40e688e095310d5f3eb5c05c28aa354 172.22.0.16:6379@16379 slave 51e77e699980423a34a6445adcb2de43b4d6022e 0 1591930968550 6 connected # 从
# 11. 查看当前节点集群信息
127.0.0.1:6379> CLUSTER INFO
cluster_state:ok # 集群状态正常
cluster_slots_assigned:16384 # 16384个哈希槽位
cluster_slots_ok:16384 # 哈希槽位全部正常
cluster_slots_pfail:0
cluster_slots_fail:0
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:6
cluster_my_epoch:1
cluster_stats_messages_ping_sent:44
cluster_stats_messages_pong_sent:45
cluster_stats_messages_sent:89
cluster_stats_messages_ping_received:40
cluster_stats_messages_pong_received:44
cluster_stats_messages_meet_received:5
cluster_stats_messages_received:89
# 12. 设置一个key
127.0.0.1:6379> set k1 v1 # 注意，此时在127.0.0.1:6379，回车之后会改变
-> Redirected to slot [12706] located at 172.22.0.13:6379 # 会发现，此时重定向到了哈希槽位12706，对应3号节点
OK
# 13. 退出客户端
172.22.0.13:6379> exit
# 14. 退出容器
root@dc086642a7fe:/data# exit
exit
# 15. 刚才我们设置key在3号节点上，3号节点是主节点，我们现在把3号节点停了。看看会发生什么？
[root@localhost ~]# docker stop redis-3
redis-3
# 16. 再次进入1号容器
[root@localhost ~]# docker exec -it redis-1 bash
# 17. 再次使用-v集群模式启动
root@dc086642a7fe:/data# redis-cli -c
# 18. 获取一下我们刚才设置的key
127.0.0.1:6379> get k1
-> Redirected to slot [12706] located at 172.22.0.14:6379 # 此时还是重定向到了槽位12706，但此时是转到了4号节点
"v1"
# 19. 我们再看一下节点信息，会发生3号节点挂了，4号节点由之前的从机转为主机了。
172.22.0.14:6379> CLUSTER NODES
55e20690b236d9e2b00e91d53ac8919732d01597 172.22.0.15:6379@16379 slave c75c0d59d8749a60300f75d13f1caff15483297c 0 1591931044989 5 connected
572ecc74c40e688e095310d5f3eb5c05c28aa354 172.22.0.16:6379@16379 slave 51e77e699980423a34a6445adcb2de43b4d6022e 0 1591931044484 6 connected
2716059d756823bdb745896d81b0d160263acd2f 172.22.0.13:6379@16379 master,fail - 1591931005157 1591931002636 3 connected # 主机挂掉了
c75c0d59d8749a60300f75d13f1caff15483297c 172.22.0.11:6379@16379 master - 0 1591931043479 1 connected 0-5460
51e77e699980423a34a6445adcb2de43b4d6022e 172.22.0.12:6379@16379 master - 0 1591931043981 2 connected 5461-10922
6d538989252611041d8410fd90aabec0f237e00a 172.22.0.14:6379@16379 myself,master - 0 1591931044000 7 connected 10923-16383 # 从机转为主机

# 20. 补充一下
# ===> 3是主，4是3的从；   停止3，4由从变主；   此时再停止4，也就是3,4都停了，此时整个集群不可用
# ===> 此时整个集群，任何命令都不可用，因为 ok的槽点不是 16384（反之，只要槽点16384的时候，整个集群才是可用的）
[root@localhost ~]# docker stop redis-4
redis-4
[root@localhost ~]# 
[root@localhost ~]# 
[root@localhost ~]# 
[root@localhost ~]# docker exec -it redis-1 bash
root@0d94e670d074:/data# redis-cli -c
127.0.0.1:6379> get k1
(error) CLUSTERDOWN The cluster is down
127.0.0.1:6379> 
127.0.0.1:6379> 
127.0.0.1:6379> set k2 v2
(error) CLUSTERDOWN The cluster is down
127.0.0.1:6379> 
127.0.0.1:6379> 
127.0.0.1:6379> CLUSTER INFO
cluster_state:fail
cluster_slots_assigned:16384
cluster_slots_ok:10923 # 只要 ok不是 16384，整个集群就不可用
cluster_slots_pfail:0
cluster_slots_fail:5461
cluster_known_nodes:6
cluster_size:3
cluster_current_epoch:7
cluster_my_epoch:1
cluster_stats_messages_ping_sent:1664
cluster_stats_messages_pong_sent:1658
cluster_stats_messages_fail_sent:10
cluster_stats_messages_auth-ack_sent:1
cluster_stats_messages_sent:3333
cluster_stats_messages_ping_received:1653
cluster_stats_messages_pong_received:1662
cluster_stats_messages_meet_received:5
cluster_stats_messages_fail_received:3
cluster_stats_messages_auth-req_received:1
cluster_stats_messages_received:3324
total_cluster_links_buffer_limit_exceeded:0

代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
# 创建自定义网络
docker network create redis --subnet 172.22.0.0/16 --gateway 172.22.0.1

# for循环6次的代码
for n in $(seq 1 6); \
do \
mkdir -p /app/docker/redis/cluster/node-${n}/conf
cat > /app/docker/redis/cluster/node-${n}/conf/redis.conf << EOF
port 6379
cluster-enabled yes
cluster-config-file nodes.conf
cluster-node-timeout 5000
cluster-announce-ip 172.22.0.1${n}
cluster-announce-port 6379
cluster-announce-bus-port 16379
appendonly yes
EOF

docker run --name redis-${n} -p 637${n}:6379 -p 1637${n}:16379 \
-v /app/docker/redis/cluster/node-${n}/data:/data \
-v /app/docker/redis/cluster/node-${n}/conf/redis.conf:/usr/local/etc/redis/redis.conf \
--net redis --ip 172.22.0.1${n} -d redis redis-server /usr/local/etc/redis/redis.conf

done

# 创建集群的代码（6个节点，1个副本，也就是3主3从）
redis-cli --cluster create 172.22.0.11:6379 172.22.0.12:6379 172.22.0.13:6379 \
172.22.0.14:6379 172.22.0.15:6379 172.22.0.16:6379 --cluster-replicas 1

# 以集群默认启动client端，加上-c
redis-cli -c

# 了解
        # 重新分片
        redis-cli --cluster reshard 127.0.0.1:7000
        redis-cli --cluster check 127.0.0.1:7000
        redis-cli reshard <host>:<port> --cluster-from <node-id> --cluster-to <node-id> --cluster-slots <number of slots> --cluster-yes
    
        # 添加节点
        redis-cli --cluster add-node 127.0.0.1:7006 127.0.0.1:7000
        redis-cli --cluster add-node 127.0.0.1:7006 127.0.0.1:7000 --cluster-slave
        redis-cli --cluster add-node 127.0.0.1:7006 127.0.0.1:7000 --cluster-slave --cluster-master-id 3c3a0c74aae0b56170ccb03a76b60cfe7dc1912e

        # 删除节点
        redis-cli --cluster del-node 127.0.0.1:7000 `<node-id>`

        # redis-cli --cluster fix命令来修复群集，以便将根据每个节点是否具有权威性的哈希槽迁移密钥。
        # 最后使用redis-cli --cluster check以确保您的群集正常。

# 强制删除6个容器
docker rm -f  redis-1  redis-2  redis-3  redis-4  redis-5  redis-6

# 强制删除本地的cluster文件夹
rm -rf /app/docker/redis/cluster/