1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.chaos.actions;
20
21 import java.io.IOException;
22 import java.util.ArrayList;
23 import java.util.Collection;
24 import java.util.HashSet;
25 import java.util.LinkedList;
26 import java.util.List;
27 import java.util.Set;
28
29 import org.apache.commons.lang.math.RandomUtils;
30 import org.apache.commons.logging.Log;
31 import org.apache.commons.logging.LogFactory;
32 import org.apache.hadoop.conf.Configuration;
33 import org.apache.hadoop.hbase.ClusterStatus;
34 import org.apache.hadoop.hbase.HBaseCluster;
35 import org.apache.hadoop.hbase.HRegionInfo;
36 import org.apache.hadoop.hbase.IntegrationTestingUtility;
37 import org.apache.hadoop.hbase.ServerLoad;
38 import org.apache.hadoop.hbase.ServerName;
39 import org.apache.hadoop.hbase.chaos.monkies.PolicyBasedChaosMonkey;
40 import org.apache.hadoop.hbase.client.Admin;
41 import org.apache.hadoop.hbase.util.Bytes;
42
43
44
45
46 public class Action {
47
48 public static final String KILL_MASTER_TIMEOUT_KEY =
49 "hbase.chaosmonkey.action.killmastertimeout";
50 public static final String START_MASTER_TIMEOUT_KEY =
51 "hbase.chaosmonkey.action.startmastertimeout";
52 public static final String KILL_RS_TIMEOUT_KEY = "hbase.chaosmonkey.action.killrstimeout";
53 public static final String START_RS_TIMEOUT_KEY = "hbase.chaosmonkey.action.startrstimeout";
54 public static final String KILL_ZK_NODE_TIMEOUT_KEY =
55 "hbase.chaosmonkey.action.killzknodetimeout";
56 public static final String START_ZK_NODE_TIMEOUT_KEY =
57 "hbase.chaosmonkey.action.startzknodetimeout";
58 public static final String KILL_DATANODE_TIMEOUT_KEY =
59 "hbase.chaosmonkey.action.killdatanodetimeout";
60 public static final String START_DATANODE_TIMEOUT_KEY =
61 "hbase.chaosmonkey.action.startdatanodetimeout";
62
63 protected static final Log LOG = LogFactory.getLog(Action.class);
64
65 protected static final long KILL_MASTER_TIMEOUT_DEFAULT = PolicyBasedChaosMonkey.TIMEOUT;
66 protected static final long START_MASTER_TIMEOUT_DEFAULT = PolicyBasedChaosMonkey.TIMEOUT;
67 protected static final long KILL_RS_TIMEOUT_DEFAULT = PolicyBasedChaosMonkey.TIMEOUT;
68 protected static final long START_RS_TIMEOUT_DEFAULT = PolicyBasedChaosMonkey.TIMEOUT;
69 protected static final long KILL_ZK_NODE_TIMEOUT_DEFAULT = PolicyBasedChaosMonkey.TIMEOUT;
70 protected static final long START_ZK_NODE_TIMEOUT_DEFAULT = PolicyBasedChaosMonkey.TIMEOUT;
71 protected static final long KILL_DATANODE_TIMEOUT_DEFAULT = PolicyBasedChaosMonkey.TIMEOUT;
72 protected static final long START_DATANODE_TIMEOUT_DEFAULT = PolicyBasedChaosMonkey.TIMEOUT;
73
74 protected ActionContext context;
75 protected HBaseCluster cluster;
76 protected ClusterStatus initialStatus;
77 protected ServerName[] initialServers;
78
79 protected long killMasterTimeout;
80 protected long startMasterTimeout;
81 protected long killRsTimeout;
82 protected long startRsTimeout;
83 protected long killZkNodeTimeout;
84 protected long startZkNodeTimeout;
85 protected long killDataNodeTimeout;
86 protected long startDataNodeTimeout;
87
88 public void init(ActionContext context) throws IOException {
89 this.context = context;
90 cluster = context.getHBaseCluster();
91 initialStatus = cluster.getInitialClusterStatus();
92 Collection<ServerName> regionServers = initialStatus.getServers();
93 initialServers = regionServers.toArray(new ServerName[regionServers.size()]);
94
95 killMasterTimeout = cluster.getConf().getLong(KILL_MASTER_TIMEOUT_KEY,
96 KILL_MASTER_TIMEOUT_DEFAULT);
97 startMasterTimeout = cluster.getConf().getLong(START_MASTER_TIMEOUT_KEY,
98 START_MASTER_TIMEOUT_DEFAULT);
99 killRsTimeout = cluster.getConf().getLong(KILL_RS_TIMEOUT_KEY, KILL_RS_TIMEOUT_DEFAULT);
100 startRsTimeout = cluster.getConf().getLong(START_RS_TIMEOUT_KEY, START_RS_TIMEOUT_DEFAULT);
101 killZkNodeTimeout = cluster.getConf().getLong(KILL_ZK_NODE_TIMEOUT_KEY,
102 KILL_ZK_NODE_TIMEOUT_DEFAULT);
103 startZkNodeTimeout = cluster.getConf().getLong(START_ZK_NODE_TIMEOUT_KEY,
104 START_ZK_NODE_TIMEOUT_DEFAULT);
105 killDataNodeTimeout = cluster.getConf().getLong(KILL_DATANODE_TIMEOUT_KEY,
106 KILL_DATANODE_TIMEOUT_DEFAULT);
107 startDataNodeTimeout = cluster.getConf().getLong(START_DATANODE_TIMEOUT_KEY,
108 START_DATANODE_TIMEOUT_DEFAULT);
109 }
110
111 public void perform() throws Exception { }
112
113
114 protected ServerName[] getCurrentServers() throws IOException {
115 ClusterStatus clusterStatus = cluster.getClusterStatus();
116 Collection<ServerName> regionServers = clusterStatus.getServers();
117 int count = regionServers == null ? 0 : regionServers.size();
118 if (count <= 0) {
119 return new ServerName [] {};
120 }
121 ServerName master = clusterStatus.getMaster();
122 Set<ServerName> masters = new HashSet<ServerName>();
123 masters.add(master);
124 masters.addAll(clusterStatus.getBackupMasters());
125 ArrayList<ServerName> tmp = new ArrayList<>(count);
126 tmp.addAll(regionServers);
127 tmp.removeAll(masters);
128 return tmp.toArray(new ServerName[tmp.size()]);
129 }
130
131 protected void killMaster(ServerName server) throws IOException {
132 LOG.info("Killing master:" + server);
133 cluster.killMaster(server);
134 cluster.waitForMasterToStop(server, killMasterTimeout);
135 LOG.info("Killed master server:" + server);
136 }
137
138 protected void startMaster(ServerName server) throws IOException {
139 LOG.info("Starting master:" + server.getHostname());
140 cluster.startMaster(server.getHostname(), server.getPort());
141 cluster.waitForActiveAndReadyMaster(startMasterTimeout);
142 LOG.info("Started master: " + server);
143 }
144
145 protected void killRs(ServerName server) throws IOException {
146 LOG.info("Killing region server:" + server);
147 cluster.killRegionServer(server);
148 cluster.waitForRegionServerToStop(server, killRsTimeout);
149 LOG.info("Killed region server:" + server + ". Reported num of rs:"
150 + cluster.getClusterStatus().getServersSize());
151 }
152
153 protected void startRs(ServerName server) throws IOException {
154 LOG.info("Starting region server:" + server.getHostname());
155 cluster.startRegionServer(server.getHostname(), server.getPort());
156 cluster.waitForRegionServerToStart(server.getHostname(), server.getPort(), startRsTimeout);
157 LOG.info("Started region server:" + server + ". Reported num of rs:"
158 + cluster.getClusterStatus().getServersSize());
159 }
160
161 protected void killZKNode(ServerName server) throws IOException {
162 LOG.info("Killing zookeeper node:" + server);
163 cluster.killZkNode(server);
164 cluster.waitForZkNodeToStop(server, killZkNodeTimeout);
165 LOG.info("Killed zookeeper node:" + server + ". Reported num of rs:"
166 + cluster.getClusterStatus().getServersSize());
167 }
168
169 protected void startZKNode(ServerName server) throws IOException {
170 LOG.info("Starting zookeeper node:" + server.getHostname());
171 cluster.startZkNode(server.getHostname(), server.getPort());
172 cluster.waitForZkNodeToStart(server, startZkNodeTimeout);
173 LOG.info("Started zookeeper node:" + server);
174 }
175
176 protected void killDataNode(ServerName server) throws IOException {
177 LOG.info("Killing datanode:" + server);
178 cluster.killDataNode(server);
179 cluster.waitForDataNodeToStop(server, killDataNodeTimeout);
180 LOG.info("Killed datanode:" + server + ". Reported num of rs:"
181 + cluster.getClusterStatus().getServersSize());
182 }
183
184 protected void startDataNode(ServerName server) throws IOException {
185 LOG.info("Starting datanode:" + server.getHostname());
186 cluster.startDataNode(server);
187 cluster.waitForDataNodeToStart(server, startDataNodeTimeout);
188 LOG.info("Started datanode:" + server);
189 }
190
191 protected void unbalanceRegions(ClusterStatus clusterStatus,
192 List<ServerName> fromServers, List<ServerName> toServers,
193 double fractionOfRegions) throws Exception {
194 List<byte[]> victimRegions = new LinkedList<byte[]>();
195 for (ServerName server : fromServers) {
196 ServerLoad serverLoad = clusterStatus.getLoad(server);
197
198 List<byte[]> regions = new LinkedList<byte[]>(serverLoad.getRegionsLoad().keySet());
199 int victimRegionCount = (int)Math.ceil(fractionOfRegions * regions.size());
200 LOG.debug("Removing " + victimRegionCount + " regions from " + server.getServerName());
201 for (int i = 0; i < victimRegionCount; ++i) {
202 int victimIx = RandomUtils.nextInt(regions.size());
203 String regionId = HRegionInfo.encodeRegionName(regions.remove(victimIx));
204 victimRegions.add(Bytes.toBytes(regionId));
205 }
206 }
207
208 LOG.info("Moving " + victimRegions.size() + " regions from " + fromServers.size()
209 + " servers to " + toServers.size() + " different servers");
210 Admin admin = this.context.getHBaseIntegrationTestingUtility().getHBaseAdmin();
211 for (byte[] victimRegion : victimRegions) {
212
213
214 if (context.isStopping()) {
215 break;
216 }
217 int targetIx = RandomUtils.nextInt(toServers.size());
218 admin.move(victimRegion, Bytes.toBytes(toServers.get(targetIx).getServerName()));
219 }
220 }
221
222 protected void forceBalancer() throws Exception {
223 Admin admin = this.context.getHBaseIntegrationTestingUtility().getHBaseAdmin();
224 boolean result = false;
225 try {
226 result = admin.balancer();
227 } catch (Exception e) {
228 LOG.warn("Got exception while doing balance ", e);
229 }
230 if (!result) {
231 LOG.error("Balancer didn't succeed");
232 }
233 }
234
235 public Configuration getConf() {
236 return cluster.getConf();
237 }
238
239
240
241
242 public static class ActionContext {
243 private IntegrationTestingUtility util;
244
245 public ActionContext(IntegrationTestingUtility util) {
246 this.util = util;
247 }
248
249 public IntegrationTestingUtility getHBaseIntegrationTestingUtility() {
250 return util;
251 }
252
253 public HBaseCluster getHBaseCluster() {
254 return util.getHBaseClusterInterface();
255 }
256
257 public boolean isStopping() {
258 return false;
259 }
260 }
261 }