上上周和同事(龙哥)参加了360组织的互联网技术训练营第三期，美团网的DBA负责人侯军伟给大家介绍了美团网在redis上踩得一些坑，讲的都是干货和坑。

Java代码    

1. package com.sohu.cache.inspect.impl;  
2.   
3. import com.sohu.cache.alert.impl.BaseAlertService;  
4. import com.sohu.cache.entity.InstanceInfo;  
5. import com.sohu.cache.inspect.InspectParamEnum;  
6. import com.sohu.cache.inspect.Inspector;  
7. import com.sohu.cache.util.IdempotentConfirmer;  
8. import com.sohu.cache.util.TypeUtil;  
9. import org.apache.commons.collections.MapUtils;  
10. import org.apache.commons.lang.StringUtils;  
11. import redis.clients.jedis.Jedis;  
12.   
13. import java.util.Collections;  
14. import java.util.LinkedHashMap;  
15. import java.util.List;  
16. import java.util.Map;  
17. import java.util.concurrent.TimeUnit;  
18.   
19.   
20. public class RedisIsolationPersistenceInspector extends BaseAlertService implements Inspector {  
21.   
22.     public static final int REDIS_DEFAULT_TIME = 5000;  
23.   
24.     @Override  
25.     public boolean inspect(Map<InspectParamEnum, Object> paramMap) {  
26.         // 某台机器和机器下所有redis实例  
27.         final String host = MapUtils.getString(paramMap, InspectParamEnum.SPLIT_KEY);  
28.         List<InstanceInfo> list = (List<InstanceInfo>) paramMap.get(InspectParamEnum.INSTANCE_LIST);  
29.         // 遍历所有的redis实例  
30.         for (InstanceInfo info : list) {  
31.             final int port = info.getPort();  
32.             final int type = info.getType();  
33.             int status = info.getStatus();  
34.             // 非正常节点  
35.             if (status != 1) {  
36.                 continue;  
37.             }  
38.             if (TypeUtil.isRedisDataType(type)) {  
39.                 Jedis jedis = new Jedis(host, port, REDIS_DEFAULT_TIME);  
40.                 try {  
41.                     // 从redis info中索取持久化信息  
42.                     Map<String, String> persistenceMap = parseMap(jedis);  
43.                     if (persistenceMap.isEmpty()) {  
44.                         logger.error("{}:{} get persistenceMap failed", host, port);  
45.                         continue;  
46.                     }  
47.                     // 如果正在进行aof就不做任何操作，理论上要等待它完毕，否则  
48.                     if (!isAofEnabled(persistenceMap)) {  
49.                         continue;  
50.                     }  
51.                     // 上一次aof重写后的尺寸和当前aof的尺寸  
52.                     long aofCurrentSize = MapUtils.getLongValue(persistenceMap, "aof_current_size");  
53.                     long aofBaseSize = MapUtils.getLongValue(persistenceMap, "aof_base_size");  
54.                     // 阀值大于60%  
55.                     long aofThresholdSize = (long) (aofBaseSize * 1.6);  
56.                     double percentage = getPercentage(aofCurrentSize, aofBaseSize);  
57.                     // 大于60%且超过60M  
58.                     if (aofCurrentSize >= aofThresholdSize && aofCurrentSize > (64 * 1024 * 1024)) {  
59.                         // bgRewriteAof 异步操作。  
60.                         boolean isInvoke = invokeBgRewriteAof(jedis);  
61.                         if (!isInvoke) {  
62.                             logger.error("{}:{} invokeBgRewriteAof failed", host, port);  
63.                             continue;  
64.                         } else {  
65.                             logger.warn("{}:{} invokeBgRewriteAof started percentage={}", host, port, percentage);  
66.                         }  
67.                         // 等待Aof重写成功（bgRewriteAof是异步操作）  
68.                         while (true) {  
69.                             try {  
70.                                 // before wait 1s  
71.                                 TimeUnit.SECONDS.sleep(1);  
72.                                 Map<String, String> loopMap = parseMap(jedis);  
73.                                 Integer aofRewriteInProgress = MapUtils.getInteger(loopMap, "aof_rewrite_in_progress", null);  
74.                                 if (aofRewriteInProgress == null) {  
75.                                     logger.error("loop watch:{}:{} return failed", host, port);  
76.                                     break;  
77.                                 } else if (aofRewriteInProgress <= 0) {  
78.                                     // bgrewriteaof Done  
79.                                     logger.warn("{}:{} bgrewriteaof Done lastSize:{}Mb,currentSize:{}Mb", host, port,  
80.                                             getMb(aofCurrentSize),  
81.                                             getMb(MapUtils.getLongValue(loopMap, "aof_current_size")));  
82.                                     break;  
83.                                 } else {  
84.                                     // wait 1s  
85.                                     TimeUnit.SECONDS.sleep(1);  
86.                                 }  
87.                             } catch (Exception e) {  
88.                                 logger.error(e.getMessage(), e);  
89.                             }  
90.                         }  
91.                     } else {  
92.                         if (percentage > 50D) {  
93.                             long currentSize = getMb(aofCurrentSize);  
94.                             logger.info("checked {}:{} aof increase percentage:{}% currentSize:{}Mb", host, port,  
95.                                     percentage, currentSize > 0 ? currentSize : "<1");  
96.                         }  
97.                     }  
98.                 } finally {  
99.                     jedis.close();  
100.                 }  
101.             }  
102.         }  
103.         return true;  
104.     }  
105.   
106.     private long getMb(long bytes) {  
107.         return (long) (bytes / 1024 / 1024);  
108.     }  
109.   
110.     private boolean isAofEnabled(Map<String, String> infoMap) {  
111.         Integer aofEnabled = MapUtils.getInteger(infoMap, "aof_enabled", null);  
112.         return aofEnabled != null && aofEnabled == 1;  
113.     }  
114.   
115.     private double getPercentage(long aofCurrentSize, long aofBaseSize) {  
116.         if (aofBaseSize == 0) {  
117.             return 0.0D;  
118.         }  
119.         String format = String.format("%.2f", (Double.valueOf(aofCurrentSize - aofBaseSize) * 100 / aofBaseSize));  
120.         return Double.parseDouble(format);  
121.     }  
122.   
123.     private Map<String, String> parseMap(final Jedis jedis) {  
124.         final StringBuilder builder = new StringBuilder();  
125.         boolean isInfo = new IdempotentConfirmer() {  
126.             @Override  
127.             public boolean execute() {  
128.                 String persistenceInfo = null;  
129.                 try {  
130.                     persistenceInfo = jedis.info("Persistence");  
131.                 } catch (Exception e) {  
132.                     logger.warn(e.getMessage() + "-{}:{}", jedis.getClient().getHost(), jedis.getClient().getPort(),  
133.                             e.getMessage());  
134.                 }  
135.                 boolean isOk = StringUtils.isNotBlank(persistenceInfo);  
136.                 if (isOk) {  
137.                     builder.append(persistenceInfo);  
138.                 }  
139.                 return isOk;  
140.             }  
141.         }.run();  
142.         if (!isInfo) {  
143.             logger.error("{}:{} info Persistence failed", jedis.getClient().getHost(), jedis.getClient().getPort());  
144.             return Collections.emptyMap();  
145.         }  
146.         String persistenceInfo = builder.toString();  
147.         if (StringUtils.isBlank(persistenceInfo)) {  
148.             return Collections.emptyMap();  
149.         }  
150.         Map<String, String> map = new LinkedHashMap<String, String>();  
151.         String[] array = persistenceInfo.split("\r\n");  
152.         for (String line : array) {  
153.             String[] cells = line.split(":");  
154.             if (cells.length > 1) {  
155.                 map.put(cells[0], cells[1]);  
156.             }  
157.         }  
158.   
159.         return map;  
160.     }  
161.   
162.     public boolean invokeBgRewriteAof(final Jedis jedis) {  
163.         return new IdempotentConfirmer() {  
164.             @Override  
165.             public boolean execute() {  
166.                 try {  
167.                     String response = jedis.bgrewriteaof();  
168.                     if (response != null && response.contains("rewriting started")) {  
169.                         return true;  
170.                     }  
171.                 } catch (Exception e) {  
172.                     String message = e.getMessage();  
173.                     if (message.contains("rewriting already")) {  
174.                         return true;  
175.                     }  
176.                     logger.error(message, e);  
177.                 }  
178.                 return false;  
179.             }  
180.         }.run();  
181.     }  
182. }